{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.使用 lightGBM 预测音乐推荐结果     模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import pickle as pk\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "from lightgbm.sklearn import LGBMRegressor\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../Data/'  # 文件路径\n",
    "model_path = '../model/' # 模型路径"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 载入训练数据\n",
    "with open(model_path + 'data_all_train_v4.pkl','rb') as fr:\n",
    "    train_X = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 载入标签数据\n",
    "with open(model_path + 'target_list.pkl','rb') as fr:\n",
    "    train_Y = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377418, 181)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377418,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 7377418 entries, 201969 to 109449\n",
      "Columns: 181 entries, source_system_tab to genre_id_166\n",
      "dtypes: category(14), uint8(167)\n",
      "memory usage: 1.4 GB\n"
     ]
    }
   ],
   "source": [
    "train_X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_length</th>\n",
       "      <th>language</th>\n",
       "      <th>reg_interval_days</th>\n",
       "      <th>...</th>\n",
       "      <th>genre_id_157</th>\n",
       "      <th>genre_id_158</th>\n",
       "      <th>genre_id_159</th>\n",
       "      <th>genre_id_160</th>\n",
       "      <th>genre_id_161</th>\n",
       "      <th>genre_id_162</th>\n",
       "      <th>genre_id_163</th>\n",
       "      <th>genre_id_164</th>\n",
       "      <th>genre_id_165</th>\n",
       "      <th>genre_id_166</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201969</th>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932462</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183559</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149511</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74867</th>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 181 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        source_system_tab   source_screen_name      source_type city bd  \\\n",
       "201969            explore              Explore  online-playlist    1  1   \n",
       "1932462        my library  Local playlist more   local-playlist   13  2   \n",
       "183559         my library  Local playlist more   local-playlist   13  2   \n",
       "149511         my library  Local playlist more   local-playlist   13  2   \n",
       "74867             explore              Explore  online-playlist    1  1   \n",
       "\n",
       "             gender registered_via song_length language reg_interval_days  \\\n",
       "201969   gender_NaN              7           1       52                 7   \n",
       "1932462      female              9           1       52                 8   \n",
       "183559       female              9           1       52                 8   \n",
       "149511       female              9           1       -1                 8   \n",
       "74867    gender_NaN              7           1       52                 7   \n",
       "\n",
       "         ... genre_id_157 genre_id_158 genre_id_159 genre_id_160  \\\n",
       "201969   ...            0            0            0            0   \n",
       "1932462  ...            0            0            0            0   \n",
       "183559   ...            0            0            0            0   \n",
       "149511   ...            0            0            0            0   \n",
       "74867    ...            0            0            0            0   \n",
       "\n",
       "         genre_id_161  genre_id_162  genre_id_163  genre_id_164  genre_id_165  \\\n",
       "201969              0             0             0             0             0   \n",
       "1932462             0             0             0             0             0   \n",
       "183559              0             0             0             0             0   \n",
       "149511              0             0             0             0             0   \n",
       "74867               0             0             0             0             0   \n",
       "\n",
       "         genre_id_166  \n",
       "201969              0  \n",
       "1932462             0  \n",
       "183559              0  \n",
       "149511              0  \n",
       "74867               0  \n",
       "\n",
       "[5 rows x 181 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存特征名字以备后用（可视化）\n",
    "column_names = train_X.columns \n",
    "\n",
    "# 有部分特征是稀疏的，有的特征是非稀疏的\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "#可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "# from scipy.sparse import csr_matrix\n",
    "# train_X = csr_matrix(train_X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 标签编码 每个分类转换成数字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将连续型变量离散化转为类别变量\n",
    "# 将原有的str 类型的变量转换为类别变量\n",
    "# with open(model_path+'data_all_train_v3.pkl', 'rb') as fr:\n",
    "#     data_all_train_v3 = pk.load(fr)\n",
    "# fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(data_all_train_v3.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7377418"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X['language'] = train_X.language.apply(lambda language_item : 'language_'+str(language_item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X['artist_name'] = train_X['artist_name'].cat.add_categories(['artist_name_NaN']);\n",
    "train_X['artist_name'] = train_X.artist_name.fillna('artist_name_NaN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X['lyricist'] = train_X['lyricist'].cat.add_categories(['lyricist_NaN']);\n",
    "train_X['lyricist'] = train_X.lyricist.fillna('lyricist_NaN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X['composer'] = train_X['composer'].cat.add_categories(['composer_NaN']);\n",
    "train_X['composer'] = train_X.composer.fillna('composer_NaN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X['name'] = train_X['name'].cat.add_categories(['name_NaN']);\n",
    "train_X['name'] = train_X.name.fillna('name_NaN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_length</th>\n",
       "      <th>language</th>\n",
       "      <th>reg_interval_days</th>\n",
       "      <th>...</th>\n",
       "      <th>genre_id_157</th>\n",
       "      <th>genre_id_158</th>\n",
       "      <th>genre_id_159</th>\n",
       "      <th>genre_id_160</th>\n",
       "      <th>genre_id_161</th>\n",
       "      <th>genre_id_162</th>\n",
       "      <th>genre_id_163</th>\n",
       "      <th>genre_id_164</th>\n",
       "      <th>genre_id_165</th>\n",
       "      <th>genre_id_166</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201969</th>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932462</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183559</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149511</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>language_-1.0</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74867</th>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 181 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        source_system_tab   source_screen_name      source_type city bd  \\\n",
       "201969            explore              Explore  online-playlist    1  1   \n",
       "1932462        my library  Local playlist more   local-playlist   13  2   \n",
       "183559         my library  Local playlist more   local-playlist   13  2   \n",
       "149511         my library  Local playlist more   local-playlist   13  2   \n",
       "74867             explore              Explore  online-playlist    1  1   \n",
       "\n",
       "             gender registered_via song_length       language  \\\n",
       "201969   gender_NaN              7           1  language_52.0   \n",
       "1932462      female              9           1  language_52.0   \n",
       "183559       female              9           1  language_52.0   \n",
       "149511       female              9           1  language_-1.0   \n",
       "74867    gender_NaN              7           1  language_52.0   \n",
       "\n",
       "        reg_interval_days  ... genre_id_157 genre_id_158 genre_id_159  \\\n",
       "201969                  7  ...            0            0            0   \n",
       "1932462                 8  ...            0            0            0   \n",
       "183559                  8  ...            0            0            0   \n",
       "149511                  8  ...            0            0            0   \n",
       "74867                   7  ...            0            0            0   \n",
       "\n",
       "        genre_id_160  genre_id_161  genre_id_162  genre_id_163  genre_id_164  \\\n",
       "201969             0             0             0             0             0   \n",
       "1932462            0             0             0             0             0   \n",
       "183559             0             0             0             0             0   \n",
       "149511             0             0             0             0             0   \n",
       "74867              0             0             0             0             0   \n",
       "\n",
       "         genre_id_165  genre_id_166  \n",
       "201969              0             0  \n",
       "1932462             0             0  \n",
       "183559              0             0  \n",
       "149511              0             0  \n",
       "74867               0             0  \n",
       "\n",
       "[5 rows x 181 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 载入测试数据\n",
    "with open(model_path + 'data_all_test_v4.pkl','rb') as fr:\n",
    "    test_X = pk.load(fr)\n",
    "fr.close()\n",
    "test_X['language'] = test_X.language.apply(lambda language_item : 'language_'+str(language_item))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 载入测试数据\n",
    "with open(model_path + 'data_all_test_v4.pkl','rb') as fr:\n",
    "    test_X = pk.load(fr)\n",
    "fr.close()\n",
    "test_X['language'] = test_X.language.apply(lambda language_item : 'language_'+str(language_item))\n",
    "test_X['artist_name'] = test_X['artist_name'].cat.add_categories(['artist_name_NaN'])\n",
    "test_X['artist_name'] = test_X.artist_name.fillna('artist_name_NaN')\n",
    "test_X['lyricist'] = test_X['lyricist'].cat.add_categories(['lyricist_NaN']);\n",
    "test_X['lyricist'] = test_X.lyricist.fillna('lyricist_NaN')\n",
    "test_X['composer'] = test_X['composer'].cat.add_categories(['composer_NaN']);\n",
    "test_X['composer'] = test_X.composer.fillna('composer_NaN')\n",
    "test_X['name'] = test_X['name'].cat.add_categories(['name_NaN']);\n",
    "test_X['name'] = test_X.name.fillna('name_NaN')\n",
    "# 保存测试数据所有的类别特征\n",
    "with open(model_path+'data_all_test_v5.pkl','wb') as fw:\n",
    "    pk.dump(test_X, fw)\n",
    "fw.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将测试数据加载\n",
    "# with open(model_path+'data_all_test_v5.pkl','rb') as fr:\n",
    "#     test_X = pk.load(fr)\n",
    "# fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_length</th>\n",
       "      <th>language</th>\n",
       "      <th>reg_interval_days</th>\n",
       "      <th>...</th>\n",
       "      <th>genre_id_157</th>\n",
       "      <th>genre_id_158</th>\n",
       "      <th>genre_id_159</th>\n",
       "      <th>genre_id_160</th>\n",
       "      <th>genre_id_161</th>\n",
       "      <th>genre_id_162</th>\n",
       "      <th>genre_id_166</th>\n",
       "      <th>genre_id_164</th>\n",
       "      <th>genre_id_163</th>\n",
       "      <th>genre_id_165</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>76126</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>language_3.0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790668</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-library</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>language_3.0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52111</th>\n",
       "      <td>discover</td>\n",
       "      <td>screen_name_NaN</td>\n",
       "      <td>song-based-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>language_17.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159296</th>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160750</th>\n",
       "      <td>radio</td>\n",
       "      <td>Radio</td>\n",
       "      <td>radio</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>language_-1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 181 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       source_system_tab   source_screen_name          source_type city bd  \\\n",
       "76126         my library  Local playlist more        local-library    1  1   \n",
       "790668        my library  Local playlist more        local-library    1  1   \n",
       "52111           discover      screen_name_NaN  song-based-playlist    1  1   \n",
       "159296             radio                Radio                radio    3  3   \n",
       "160750             radio                Radio                radio    3  3   \n",
       "\n",
       "            gender registered_via song_length       language  \\\n",
       "76126   gender_NaN              7           1   language_3.0   \n",
       "790668  gender_NaN              7           2   language_3.0   \n",
       "52111   gender_NaN              4           2  language_17.0   \n",
       "159296        male              9           1  language_52.0   \n",
       "160750        male              9           1  language_-1.0   \n",
       "\n",
       "       reg_interval_days  ... genre_id_157 genre_id_158 genre_id_159  \\\n",
       "76126                  3  ...            0            0            0   \n",
       "790668                 3  ...            0            0            0   \n",
       "52111                  0  ...            0            0            0   \n",
       "159296                10  ...            0            0            0   \n",
       "160750                10  ...            0            0            0   \n",
       "\n",
       "       genre_id_160  genre_id_161  genre_id_162  genre_id_166  genre_id_164  \\\n",
       "76126             0             0             0             0             0   \n",
       "790668            0             0             0             0             0   \n",
       "52111             0             0             0             0             0   \n",
       "159296            0             0             0             0             0   \n",
       "160750            0             0             0             0             0   \n",
       "\n",
       "        genre_id_163  genre_id_165  \n",
       "76126              0             0  \n",
       "790668             0             0  \n",
       "52111              0             0  \n",
       "159296             0             0  \n",
       "160750             0             0  \n",
       "\n",
       "[5 rows x 181 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gender_NaN', 'male', 'female', 'gender_NaN', 'male', 'female']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_temp_list = list(train_X['gender'].value_counts().index)\n",
    "column_temp_list.extend(list(test_X['gender'].value_counts().index))\n",
    "column_temp_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_length</th>\n",
       "      <th>language</th>\n",
       "      <th>reg_interval_days</th>\n",
       "      <th>...</th>\n",
       "      <th>genre_id_157</th>\n",
       "      <th>genre_id_158</th>\n",
       "      <th>genre_id_159</th>\n",
       "      <th>genre_id_160</th>\n",
       "      <th>genre_id_161</th>\n",
       "      <th>genre_id_162</th>\n",
       "      <th>genre_id_163</th>\n",
       "      <th>genre_id_164</th>\n",
       "      <th>genre_id_165</th>\n",
       "      <th>genre_id_166</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201969</th>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932462</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183559</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149511</th>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>language_-1.0</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74867</th>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gender_NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>language_52.0</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 181 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        source_system_tab   source_screen_name      source_type city bd  \\\n",
       "201969            explore              Explore  online-playlist    1  1   \n",
       "1932462        my library  Local playlist more   local-playlist   13  2   \n",
       "183559         my library  Local playlist more   local-playlist   13  2   \n",
       "149511         my library  Local playlist more   local-playlist   13  2   \n",
       "74867             explore              Explore  online-playlist    1  1   \n",
       "\n",
       "             gender registered_via song_length       language  \\\n",
       "201969   gender_NaN              7           1  language_52.0   \n",
       "1932462      female              9           1  language_52.0   \n",
       "183559       female              9           1  language_52.0   \n",
       "149511       female              9           1  language_-1.0   \n",
       "74867    gender_NaN              7           1  language_52.0   \n",
       "\n",
       "        reg_interval_days  ... genre_id_157 genre_id_158 genre_id_159  \\\n",
       "201969                  7  ...            0            0            0   \n",
       "1932462                 8  ...            0            0            0   \n",
       "183559                  8  ...            0            0            0   \n",
       "149511                  8  ...            0            0            0   \n",
       "74867                   7  ...            0            0            0   \n",
       "\n",
       "        genre_id_160  genre_id_161  genre_id_162  genre_id_163  genre_id_164  \\\n",
       "201969             0             0             0             0             0   \n",
       "1932462            0             0             0             0             0   \n",
       "183559             0             0             0             0             0   \n",
       "149511             0             0             0             0             0   \n",
       "74867              0             0             0             0             0   \n",
       "\n",
       "         genre_id_165  genre_id_166  \n",
       "201969              0             0  \n",
       "1932462             0             0  \n",
       "183559              0             0  \n",
       "149511              0             0  \n",
       "74867               0             0  \n",
       "\n",
       "[5 rows x 181 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source_system_tab\n",
      "\n",
      "source_screen_name\n",
      "\n",
      "source_type\n",
      "\n",
      "city\n",
      "\n",
      "bd\n",
      "\n",
      "gender\n",
      "\n",
      "registered_via\n",
      "\n",
      "song_length\n",
      "\n",
      "language\n",
      "\n",
      "reg_interval_days\n",
      "\n",
      "artist_name\n",
      "\n",
      "composer\n",
      "\n",
      "genre_ids\n",
      "\n",
      "lyricist\n",
      "\n",
      "name\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 所有的类别型特征都进行标签编码\n",
    "for column in ['source_system_tab', 'source_screen_name', 'source_type', 'city', 'bd',\n",
    "       'gender', 'registered_via', 'song_length', 'language',\n",
    "       'reg_interval_days', 'artist_name', 'composer', 'genre_ids', 'lyricist',\n",
    "       'name']:\n",
    "    print(column+'\\n')\n",
    "    if column not in ['genre_ids']:\n",
    "        le = LabelEncoder()\n",
    "        column_temp_list = list(train_X[column].value_counts().index)\n",
    "        column_temp_list.extend(list(test_X[column].value_counts().index))\n",
    "        le.fit(column_temp_list)\n",
    "        with open(model_path+'LE'+column+'.pkl','wb') as fw:\n",
    "            pk.dump(le,fw)\n",
    "        fw.close()\n",
    "        train_X[column] = le.transform(train_X[column])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>city</th>\n",
       "      <th>bd</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_length</th>\n",
       "      <th>language</th>\n",
       "      <th>reg_interval_days</th>\n",
       "      <th>...</th>\n",
       "      <th>genre_id_157</th>\n",
       "      <th>genre_id_158</th>\n",
       "      <th>genre_id_159</th>\n",
       "      <th>genre_id_160</th>\n",
       "      <th>genre_id_161</th>\n",
       "      <th>genre_id_162</th>\n",
       "      <th>genre_id_163</th>\n",
       "      <th>genre_id_164</th>\n",
       "      <th>genre_id_165</th>\n",
       "      <th>genre_id_166</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201969</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932462</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183559</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149511</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74867</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 181 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         source_system_tab  source_screen_name  source_type  city  bd  gender  \\\n",
       "201969                   1                   7            6     0   0       1   \n",
       "1932462                  3                   8            4    11   1       0   \n",
       "183559                   3                   8            4    11   1       0   \n",
       "149511                   3                   8            4    11   1       0   \n",
       "74867                    1                   7            6     0   0       1   \n",
       "\n",
       "         registered_via  song_length  language  reg_interval_days  ...  \\\n",
       "201969                2            0         8                  7  ...   \n",
       "1932462               3            0         8                  8  ...   \n",
       "183559                3            0         8                  8  ...   \n",
       "149511                3            0         0                  8  ...   \n",
       "74867                 2            0         8                  7  ...   \n",
       "\n",
       "         genre_id_157  genre_id_158  genre_id_159  genre_id_160  genre_id_161  \\\n",
       "201969              0             0             0             0             0   \n",
       "1932462             0             0             0             0             0   \n",
       "183559              0             0             0             0             0   \n",
       "149511              0             0             0             0             0   \n",
       "74867               0             0             0             0             0   \n",
       "\n",
       "         genre_id_162  genre_id_163  genre_id_164  genre_id_165  genre_id_166  \n",
       "201969              0             0             0             0             0  \n",
       "1932462             0             0             0             0             0  \n",
       "183559              0             0             0             0             0  \n",
       "149511              0             0             0             0             0  \n",
       "74867               0             0             0             0             0  \n",
       "\n",
       "[5 rows x 181 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3    3684730\n",
       "0    2179252\n",
       "6     623286\n",
       "5     476701\n",
       "2     212266\n",
       "1     167949\n",
       "8      24849\n",
       "4       6185\n",
       "7       2200\n",
       "Name: source_system_tab, dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.source_system_tab.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377418, 181)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "my library      1019492\n",
       "discover         871068\n",
       "search           277615\n",
       "radio            212765\n",
       "listen with       98628\n",
       "explore           66023\n",
       "tab_NaN            8442\n",
       "notification       2124\n",
       "settings            633\n",
       "Name: source_system_tab, dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_X.source_system_tab.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 7377418 entries, 201969 to 109449\n",
      "Columns: 181 entries, source_system_tab to genre_id_166\n",
      "dtypes: int64(14), uint8(167)\n",
      "memory usage: 2.0 GB\n"
     ]
    }
   ],
   "source": [
    "train_X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(model_path + 'data_all_train_lgbm_v1.pkl','wb') as fw:\n",
    "#     pk.dump(train_X,fw)\n",
    "# fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_path + 'data_all_train_lgbm_v1.pkl','rb') as fr:\n",
    "    train_X = pk.load(fr)\n",
    "fr.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM超参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LightGBM的主要的超参包括：\n",
    "1. 树的数目n_estimators 和 学习率 learning_rate\n",
    "2. 树的最大深度max_depth 和 树的最大叶子节点数目num_leaves（注意：XGBoost只有max_depth，LightGBM采用叶子优先的方式生成树，num_leaves很重要，设置成比 2^max_depth 小）\n",
    "3. 叶子结点的最小样本数:min_data_in_leaf(min_data, min_child_samples)\n",
    "4. 每棵树的列采样比例：feature_fraction/colsample_bytree\n",
    "5. 每棵树的行采样比例：bagging_fraction （需同时设置bagging_freq=1）/subsample\n",
    "6. 正则化参数lambda_l1(reg_alpha), lambda_l2(reg_lambda)\n",
    "7. 两个非模型复杂度参数，但会影响模型速度和精度。可根据特征取值范围和样本数目修改这两个参数\n",
    "1）特征的最大bin数目max_bin：默认255；\n",
    "2）用来建立直方图的样本数目subsample_for_bin：默认200000。\n",
    "\n",
    "对n_estimators，用LightGBM内嵌的cv函数调优，因为同XGBoost一样，LightGBM学习的过程内嵌了cv，速度极快。\n",
    "其他参数用GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_ROUNDS = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 相同的交叉验证分组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 5.48 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#直接调用 lightgbm 内嵌的交叉验证(cv)，可对连续的 n_estimators 参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证，且速度相对较慢\n",
    "def get_n_estimators(params , train_X , train_Y , early_stopping_rounds=10):\n",
    "    lgbm_params = params.copy()\n",
    "     \n",
    "    lgbmtrain = lgbm.Dataset(train_X , train_Y)\n",
    "     \n",
    "    #num_boost_round为弱分类器数目，下面的代码参数里因为已经设置了early_stopping_rounds\n",
    "    #即性能未提升的次数超过过早停止设置的数值，则停止训练\n",
    "    cv_result = lgbm.cv(lgbm_params , lgbmtrain , num_boost_round=MAX_ROUNDS , nfold=3,  metrics='auc' , early_stopping_rounds=early_stopping_rounds,seed=3 )\n",
    "     \n",
    "    return cv_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 为了得到 n_estimators ，需要为其他重要的参数赋一个初始值，如 params 中的值。\n",
    "初始值的意义不大，只是为了方便确定其他参数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1d 22min 7s, sys: 27min 11s, total: 1d 49min 18s\n",
      "Wall time: 4h 8min 53s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.1,\n",
    "          'num_leaves': 60,\n",
    "          'max_depth': 6,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "\n",
    "cv_result = get_n_estimators(params , train_X , train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'auc-mean': [0.6491699599180132,\n",
       "  0.6609447761885159,\n",
       "  0.6627949718357063,\n",
       "  0.6654401576134231,\n",
       "  0.6659538681180096,\n",
       "  0.6662199511597017,\n",
       "  0.666975778667546,\n",
       "  0.6680350676769905,\n",
       "  0.6685884718655366,\n",
       "  0.6690672250133639,\n",
       "  0.6696340054922111,\n",
       "  0.6704203433576481,\n",
       "  0.6707984328490063,\n",
       "  0.6711515140112034,\n",
       "  0.6716005786541189,\n",
       "  0.6718675791639969,\n",
       "  0.6720041982625404,\n",
       "  0.6724796029182302,\n",
       "  0.6726941447999407,\n",
       "  0.6728720974547938,\n",
       "  0.6730169006517345,\n",
       "  0.6731930423927088,\n",
       "  0.673355015668438,\n",
       "  0.6736295457618665,\n",
       "  0.6739240459852601,\n",
       "  0.6741850633156892,\n",
       "  0.6743099990169608,\n",
       "  0.6745818458777442,\n",
       "  0.6748746861821564,\n",
       "  0.6750121229892357,\n",
       "  0.675752105507549,\n",
       "  0.6759444127153538,\n",
       "  0.6761953337291525,\n",
       "  0.6763957260808989,\n",
       "  0.6767381943598023,\n",
       "  0.6769042355033976,\n",
       "  0.6770851854059964,\n",
       "  0.6772684172469897,\n",
       "  0.67748180696098,\n",
       "  0.6775930568908519,\n",
       "  0.6777591557045369,\n",
       "  0.6778589178279796,\n",
       "  0.6780299694005869,\n",
       "  0.6784022429270177,\n",
       "  0.678527716989478,\n",
       "  0.6787008289466776,\n",
       "  0.6789167808786578,\n",
       "  0.6790719824124033,\n",
       "  0.679362251880265,\n",
       "  0.6794805836972548,\n",
       "  0.6795491666932798,\n",
       "  0.6796885540104022,\n",
       "  0.6798178899921941,\n",
       "  0.6798810978825433,\n",
       "  0.6799905995525813,\n",
       "  0.6801013372670802,\n",
       "  0.6802136211761382,\n",
       "  0.6803284492001994,\n",
       "  0.6804591791777225,\n",
       "  0.6805501794631986,\n",
       "  0.6806600752574203,\n",
       "  0.6808127587228632,\n",
       "  0.6808857197822,\n",
       "  0.6809257940430804,\n",
       "  0.680974958264721,\n",
       "  0.6811132335807869,\n",
       "  0.6812418196123637,\n",
       "  0.6813980744307947,\n",
       "  0.6814777382265861,\n",
       "  0.6815333084270939,\n",
       "  0.6816394017616947,\n",
       "  0.6817023492200353,\n",
       "  0.6817723558488981,\n",
       "  0.6818630698546367,\n",
       "  0.6819455462396656,\n",
       "  0.682095577324155,\n",
       "  0.6821187032321051,\n",
       "  0.6822119524355964,\n",
       "  0.6823084746688491,\n",
       "  0.6823863046039218,\n",
       "  0.6825167277124118,\n",
       "  0.6825807410163861,\n",
       "  0.6826509266519021,\n",
       "  0.6827222904028162,\n",
       "  0.6828349998465643,\n",
       "  0.6829518978192354,\n",
       "  0.6829975024666731,\n",
       "  0.6830709306364539,\n",
       "  0.6831312419944752,\n",
       "  0.6831646992480774,\n",
       "  0.6832515330732284,\n",
       "  0.6833060714856701,\n",
       "  0.6833421472442353,\n",
       "  0.6834110459693634,\n",
       "  0.6834632551945777,\n",
       "  0.6836106511872212,\n",
       "  0.6836376903712771,\n",
       "  0.683701333760085,\n",
       "  0.6837502171887074,\n",
       "  0.6838370552909706,\n",
       "  0.6838934806194752,\n",
       "  0.6840971802415269,\n",
       "  0.6841565212511916,\n",
       "  0.6842120206229195,\n",
       "  0.6842888295367784,\n",
       "  0.684406747129832,\n",
       "  0.6844937378534274,\n",
       "  0.6845895685140674,\n",
       "  0.6847007412213731,\n",
       "  0.6848256021278812,\n",
       "  0.684856727133376,\n",
       "  0.6849389457665659,\n",
       "  0.6849781211281386,\n",
       "  0.6850439218442799,\n",
       "  0.6850999222201364,\n",
       "  0.6851320042015621,\n",
       "  0.685172868288794,\n",
       "  0.6852239942743088,\n",
       "  0.6852663792155113,\n",
       "  0.6853057854106644,\n",
       "  0.6853529349356755,\n",
       "  0.6854622952216974,\n",
       "  0.6854912438191031,\n",
       "  0.685586273432191,\n",
       "  0.6856941452809622,\n",
       "  0.6857429457710831,\n",
       "  0.6858195743985948,\n",
       "  0.6858846556324215,\n",
       "  0.6859293441802516,\n",
       "  0.6859811315595087,\n",
       "  0.6861011307885615,\n",
       "  0.6861209365827013,\n",
       "  0.6861536832741676,\n",
       "  0.6861847405200132,\n",
       "  0.6862041889311318,\n",
       "  0.6862701846504285,\n",
       "  0.6863742316015408,\n",
       "  0.6864511963665995,\n",
       "  0.6864987817430154,\n",
       "  0.6865734904633891,\n",
       "  0.6866897556465169,\n",
       "  0.686721837385326,\n",
       "  0.6868198740133086,\n",
       "  0.68690248888897,\n",
       "  0.6869770038508819,\n",
       "  0.6870120852816114,\n",
       "  0.6870574821624166,\n",
       "  0.6870811264278561,\n",
       "  0.6871161696253242,\n",
       "  0.6871653751631488,\n",
       "  0.6872105020853176,\n",
       "  0.6872719067559747,\n",
       "  0.6873360404012252,\n",
       "  0.687372697239728,\n",
       "  0.6874133548023355,\n",
       "  0.6874316251496925,\n",
       "  0.687480392726291,\n",
       "  0.6875210263593184,\n",
       "  0.6875615668209618,\n",
       "  0.6876162145227595,\n",
       "  0.6876932087920052,\n",
       "  0.6877413637011774,\n",
       "  0.6877942266151106,\n",
       "  0.6878600352663106,\n",
       "  0.6879237055578608,\n",
       "  0.6879878030309042,\n",
       "  0.6880079096857724,\n",
       "  0.6880589100510598,\n",
       "  0.6881058246963162,\n",
       "  0.6881333408794927,\n",
       "  0.6881676451561276,\n",
       "  0.6882179650473805,\n",
       "  0.6882495002648651,\n",
       "  0.688288727204542,\n",
       "  0.6883407318089559,\n",
       "  0.6884019607541983,\n",
       "  0.6884794215235668,\n",
       "  0.6885223172019557,\n",
       "  0.6885714042811588,\n",
       "  0.6886218607604183,\n",
       "  0.6886417213621449,\n",
       "  0.6886697572015036,\n",
       "  0.6887420670485968,\n",
       "  0.6887985791286321,\n",
       "  0.6888235449277404,\n",
       "  0.6888661138685365,\n",
       "  0.6889297046579622,\n",
       "  0.6889746654402283,\n",
       "  0.6890222245344492,\n",
       "  0.689058256278574,\n",
       "  0.6891242601077927,\n",
       "  0.6891810725229108,\n",
       "  0.689245645671214,\n",
       "  0.6892878210856409,\n",
       "  0.6893470416791375,\n",
       "  0.6893845371435101,\n",
       "  0.6894044652981101,\n",
       "  0.6894457341507847,\n",
       "  0.6895072602670208,\n",
       "  0.6895650564071844,\n",
       "  0.6896000202971709,\n",
       "  0.6896570556811351,\n",
       "  0.6896804033807373,\n",
       "  0.6896997038175424,\n",
       "  0.6897366169117215,\n",
       "  0.6897566298357504,\n",
       "  0.6897898484573646,\n",
       "  0.6898144786193336,\n",
       "  0.6898438642047616,\n",
       "  0.6898785319689876,\n",
       "  0.6899579962534785,\n",
       "  0.690006955220808,\n",
       "  0.690033509181632,\n",
       "  0.6900837733241293,\n",
       "  0.6901030791396078,\n",
       "  0.6901210216727091,\n",
       "  0.6901516471176344,\n",
       "  0.6902010256332641,\n",
       "  0.6902739220468616,\n",
       "  0.6903248116659416,\n",
       "  0.6903503857684176,\n",
       "  0.6903844342911506,\n",
       "  0.6904372530575641,\n",
       "  0.6904664832144313,\n",
       "  0.6905027622717709,\n",
       "  0.6905495292601943,\n",
       "  0.6905860109607125,\n",
       "  0.6906519165310466,\n",
       "  0.6907069762118998,\n",
       "  0.6907527125523366,\n",
       "  0.6907710762540008,\n",
       "  0.6908325781038301,\n",
       "  0.6908737220896694,\n",
       "  0.690927080472319,\n",
       "  0.6910106978763192,\n",
       "  0.6910277549650088,\n",
       "  0.6910586273276998,\n",
       "  0.6910921770023694,\n",
       "  0.6911067459349053,\n",
       "  0.6911292353410209,\n",
       "  0.6911587233673536,\n",
       "  0.6912121611513045,\n",
       "  0.6912453809064264,\n",
       "  0.6912769841465458,\n",
       "  0.6913481698226329,\n",
       "  0.69138954930237,\n",
       "  0.6914160896094496,\n",
       "  0.691448387359935,\n",
       "  0.6914819318377513,\n",
       "  0.6915537346150463,\n",
       "  0.6915866177403102,\n",
       "  0.6916344863913264,\n",
       "  0.6916722143760602,\n",
       "  0.6916904130314693,\n",
       "  0.691729380421431,\n",
       "  0.6917520818137768,\n",
       "  0.6917827389432754,\n",
       "  0.6918036658767291,\n",
       "  0.691831762451506,\n",
       "  0.6918516425215969,\n",
       "  0.6918969364208607,\n",
       "  0.691950211199086,\n",
       "  0.6919873685976482,\n",
       "  0.6920412287840918,\n",
       "  0.6920876983976355,\n",
       "  0.6921151441487092,\n",
       "  0.6921540269038274,\n",
       "  0.6921904705217597,\n",
       "  0.6922145128430138,\n",
       "  0.6922356703151662,\n",
       "  0.6923038346217428,\n",
       "  0.6923414020818192,\n",
       "  0.6923793151445596,\n",
       "  0.692417699595636,\n",
       "  0.6924430244842489,\n",
       "  0.692464298687336,\n",
       "  0.6925129885009159,\n",
       "  0.6925447596835714,\n",
       "  0.6925623006953199,\n",
       "  0.6926130801961298,\n",
       "  0.6926459918559155,\n",
       "  0.6927037676881748,\n",
       "  0.6927249944257022,\n",
       "  0.6927461718841569,\n",
       "  0.6927659426963882,\n",
       "  0.6927906766715634,\n",
       "  0.6928167728893739,\n",
       "  0.6928356353307749,\n",
       "  0.6928542856605063,\n",
       "  0.6928826343766875,\n",
       "  0.6929082600118545,\n",
       "  0.6929208097737053,\n",
       "  0.6929284386819606,\n",
       "  0.6929409875667494,\n",
       "  0.692969395224505,\n",
       "  0.6930060646960415,\n",
       "  0.6930323094362794,\n",
       "  0.6930501741278233,\n",
       "  0.6930936633540492,\n",
       "  0.693133211636641,\n",
       "  0.6931752862319526,\n",
       "  0.6932060569088759,\n",
       "  0.6932185737530862,\n",
       "  0.6932540874376398,\n",
       "  0.6932641405647523,\n",
       "  0.6933325747028668,\n",
       "  0.6933739956233583,\n",
       "  0.6933968109724282,\n",
       "  0.6934200879071621,\n",
       "  0.6934679021955183,\n",
       "  0.6934941927051742,\n",
       "  0.693510915269573,\n",
       "  0.6935277706532595,\n",
       "  0.6935643165218522,\n",
       "  0.6935834890014116,\n",
       "  0.6936092401721963,\n",
       "  0.6936264853301312,\n",
       "  0.6936551418586366,\n",
       "  0.693690336184306,\n",
       "  0.6937020390609968,\n",
       "  0.6937540150493099,\n",
       "  0.6937802492237841,\n",
       "  0.6938289822166664,\n",
       "  0.6938583625227692,\n",
       "  0.6938832788945947,\n",
       "  0.6939125543887524,\n",
       "  0.6939223573010024,\n",
       "  0.6939497605590125,\n",
       "  0.6939669166455058,\n",
       "  0.6940000041148114,\n",
       "  0.6940491106044272,\n",
       "  0.6940817412625307,\n",
       "  0.6941015246187877,\n",
       "  0.6941141026444405,\n",
       "  0.6941324414268526,\n",
       "  0.6941842151064427,\n",
       "  0.6942149955604124,\n",
       "  0.6942364979187365,\n",
       "  0.6942594362481219,\n",
       "  0.6942946251802523,\n",
       "  0.6943181046859594,\n",
       "  0.6943460839012028,\n",
       "  0.6943745552102937,\n",
       "  0.6944057513707086,\n",
       "  0.694442034508901,\n",
       "  0.6944871607966664,\n",
       "  0.6945366800680862,\n",
       "  0.6945499787448082,\n",
       "  0.694591827551541,\n",
       "  0.6946148895068062,\n",
       "  0.694643769695248,\n",
       "  0.6946815366140683,\n",
       "  0.6946991386094657,\n",
       "  0.6947553121010505,\n",
       "  0.6947790721146214,\n",
       "  0.6947923272403537,\n",
       "  0.6948370166591259,\n",
       "  0.6948701702732656,\n",
       "  0.6948885993830117,\n",
       "  0.6949091407403835,\n",
       "  0.6949197679651359,\n",
       "  0.6949508571897969,\n",
       "  0.6949706274467878,\n",
       "  0.6950070717745999,\n",
       "  0.6950249065740585,\n",
       "  0.6950584490981692,\n",
       "  0.6950924565953689,\n",
       "  0.6951291418122324,\n",
       "  0.6951432079565936,\n",
       "  0.6951988052610986,\n",
       "  0.6952090649462158,\n",
       "  0.695249638750608,\n",
       "  0.6952985245516756,\n",
       "  0.6953256917922851,\n",
       "  0.6953449611957628,\n",
       "  0.6953571558615063,\n",
       "  0.6953820268514718,\n",
       "  0.6954292455932815,\n",
       "  0.6954460383701981,\n",
       "  0.6954716841142338,\n",
       "  0.6955053683455885,\n",
       "  0.6955366850208365,\n",
       "  0.6955480009708795,\n",
       "  0.6955623410084596,\n",
       "  0.6955683334407209,\n",
       "  0.6956063312491146,\n",
       "  0.6957138471427978,\n",
       "  0.6957563289445042,\n",
       "  0.6957776379343592,\n",
       "  0.6958084275812726,\n",
       "  0.6958412047178885,\n",
       "  0.6958567803481243,\n",
       "  0.6958603985584633,\n",
       "  0.695894089838225,\n",
       "  0.695918874904002,\n",
       "  0.6959367688937861,\n",
       "  0.6959708088493035,\n",
       "  0.6960038170614178,\n",
       "  0.6960388033746896,\n",
       "  0.6960718250694912,\n",
       "  0.6960942870509842,\n",
       "  0.6961146689392583,\n",
       "  0.6961301547282503,\n",
       "  0.6961564293716415,\n",
       "  0.6961744625471303,\n",
       "  0.6962155180133438,\n",
       "  0.6962429881443666,\n",
       "  0.696267291017874,\n",
       "  0.6962956813953946,\n",
       "  0.6963108112711486,\n",
       "  0.6963517430460812,\n",
       "  0.6963873890797019,\n",
       "  0.6963927403479314,\n",
       "  0.6964163422624624,\n",
       "  0.6964507846519997,\n",
       "  0.6964811598319646,\n",
       "  0.6965023005694314,\n",
       "  0.6965258951328629,\n",
       "  0.6965480679805025,\n",
       "  0.6965760501832877,\n",
       "  0.6965892420323957,\n",
       "  0.6966050379799684,\n",
       "  0.6966549491227824,\n",
       "  0.6966725199767639,\n",
       "  0.6966848489234884,\n",
       "  0.6967019458589477,\n",
       "  0.6967170431301725,\n",
       "  0.6967262733058269,\n",
       "  0.6967481168489007,\n",
       "  0.6967691712390721,\n",
       "  0.6967846230224762,\n",
       "  0.6968191259821649,\n",
       "  0.696837054088581,\n",
       "  0.6968559414315013,\n",
       "  0.6968667817635522,\n",
       "  0.6968782302303866,\n",
       "  0.6969015335958625,\n",
       "  0.6969212125800118,\n",
       "  0.6969342647777368,\n",
       "  0.6969634632364681,\n",
       "  0.6969798272355462,\n",
       "  0.6969964126264089,\n",
       "  0.6970287847777054,\n",
       "  0.6970405286778334,\n",
       "  0.6970816484902982,\n",
       "  0.6971009790472333,\n",
       "  0.6971254337474249,\n",
       "  0.6971465563164072,\n",
       "  0.6971680108029799,\n",
       "  0.6971834351277778,\n",
       "  0.6971981667063244,\n",
       "  0.6972113440920148,\n",
       "  0.6972308181025225,\n",
       "  0.6972698310171945,\n",
       "  0.6972945897443396,\n",
       "  0.6973041631599513,\n",
       "  0.6973263239585958,\n",
       "  0.6973442881934518,\n",
       "  0.6973656740258787,\n",
       "  0.6973927132530383,\n",
       "  0.6974152445216387,\n",
       "  0.6974369871768752,\n",
       "  0.6974572463376808,\n",
       "  0.6974810568802766,\n",
       "  0.6975049641950722,\n",
       "  0.6975236453530643,\n",
       "  0.6975655032963998,\n",
       "  0.6975872196445345,\n",
       "  0.6976298931099301,\n",
       "  0.697647007986009,\n",
       "  0.6976759783684156,\n",
       "  0.6976973831249781,\n",
       "  0.6977183273792716,\n",
       "  0.697729536979891,\n",
       "  0.6977439478417228,\n",
       "  0.6977651529834921,\n",
       "  0.697783261707821,\n",
       "  0.6978119512141833,\n",
       "  0.6978308246449422,\n",
       "  0.6978404240816037,\n",
       "  0.6978478005584537,\n",
       "  0.6978591663773989,\n",
       "  0.6978701086454202,\n",
       "  0.6978855451247624,\n",
       "  0.6979065858057606,\n",
       "  0.6979201382289334,\n",
       "  0.6979418110410721,\n",
       "  0.6979577025709917,\n",
       "  0.6979682246220295,\n",
       "  0.697974133548081,\n",
       "  0.6979835313045649,\n",
       "  0.6979997930991285,\n",
       "  0.698032420855653,\n",
       "  0.6980669910475347,\n",
       "  0.6980733162403648,\n",
       "  0.6980898657404359,\n",
       "  0.6981044738735354,\n",
       "  0.6981219972603642,\n",
       "  0.6981362803071643,\n",
       "  0.6981462987892278,\n",
       "  0.6981675635500721,\n",
       "  0.698183594598154,\n",
       "  0.6982082955827563,\n",
       "  0.6982699543382808,\n",
       "  0.6983178695380423,\n",
       "  0.6983334309706447,\n",
       "  0.6983444189831861,\n",
       "  0.6983630357715235,\n",
       "  0.6983986567619396,\n",
       "  0.698408539135952,\n",
       "  0.6984584059536179,\n",
       "  0.6984943143001167,\n",
       "  0.698502874506973,\n",
       "  0.698528592307757,\n",
       "  0.6985670145135031,\n",
       "  0.6985844801923199,\n",
       "  0.6986156362602839,\n",
       "  0.6986464917120155,\n",
       "  0.698668516479561,\n",
       "  0.6986834515858856,\n",
       "  0.698692913342704,\n",
       "  0.6987177766622216,\n",
       "  0.6987323704875829,\n",
       "  0.6987513499571648,\n",
       "  0.698774861003136,\n",
       "  0.698786020710231,\n",
       "  0.6988051872082056,\n",
       "  0.6988216347462567,\n",
       "  0.6988438048264562,\n",
       "  0.6988570707178986,\n",
       "  0.6988679152592113,\n",
       "  0.6988818069007605,\n",
       "  0.69889178602136,\n",
       "  0.6989127282580759,\n",
       "  0.6989259918513397,\n",
       "  0.6989767482293118,\n",
       "  0.6989900656846252,\n",
       "  0.6990161331341133,\n",
       "  0.6990504167929302,\n",
       "  0.6990626488706236,\n",
       "  0.6990880687924325,\n",
       "  0.6991138380462271,\n",
       "  0.6991233111535147,\n",
       "  0.6991460442007792,\n",
       "  0.6991794020768598,\n",
       "  0.699188469879879,\n",
       "  0.6992092042229042,\n",
       "  0.699221279977718,\n",
       "  0.6992312529078389,\n",
       "  0.699246357370613,\n",
       "  0.6992551391696202,\n",
       "  0.6992724651863141,\n",
       "  0.6992923022170787,\n",
       "  0.699305100771418,\n",
       "  0.6993182543603295,\n",
       "  0.6993347033655217,\n",
       "  0.6993490816042991,\n",
       "  0.6993625856125864,\n",
       "  0.6993826038559131,\n",
       "  0.6994033763232643,\n",
       "  0.6994206108799125,\n",
       "  0.6994529321771922,\n",
       "  0.6994792870720037,\n",
       "  0.6994904430515536,\n",
       "  0.6995089902081634,\n",
       "  0.6995180252231123,\n",
       "  0.699539560432035,\n",
       "  0.6995531060790142,\n",
       "  0.6995823089389929,\n",
       "  0.699607430810535,\n",
       "  0.6996241029903717,\n",
       "  0.6996338745236131,\n",
       "  0.6996435348585838,\n",
       "  0.6996706059917482,\n",
       "  0.6996770267986231,\n",
       "  0.6996957618516606,\n",
       "  0.6997114964104637,\n",
       "  0.6997237978165315,\n",
       "  0.6997361107611509,\n",
       "  0.6997580423319584,\n",
       "  0.6997825357653941,\n",
       "  0.6998037691303184,\n",
       "  0.6998200469329552,\n",
       "  0.6998324190156247,\n",
       "  0.6998589757142235,\n",
       "  0.6998635654282902,\n",
       "  0.6998713197396995,\n",
       "  0.6999007050246474,\n",
       "  0.6999213008994484,\n",
       "  0.6999374410994954,\n",
       "  0.6999582311958473,\n",
       "  0.6999704057568956,\n",
       "  0.6999861455652293,\n",
       "  0.6999954255420798,\n",
       "  0.7000016931780745,\n",
       "  0.7000069667012309,\n",
       "  0.7000178642522128,\n",
       "  0.7000379073901696,\n",
       "  0.7000563375094213,\n",
       "  0.7000699921508567,\n",
       "  0.7000809460246304,\n",
       "  0.7001159921332193,\n",
       "  0.7001313263334771,\n",
       "  0.7001414349924749,\n",
       "  0.7001670324326527,\n",
       "  0.7001754569554338,\n",
       "  0.7001908429946729,\n",
       "  0.7002072776297737,\n",
       "  0.700226940352474,\n",
       "  0.7002489319385182,\n",
       "  0.7002625691187917,\n",
       "  0.7002698864211223,\n",
       "  0.7002812610952985,\n",
       "  0.7002879475010206,\n",
       "  0.7002976169879256,\n",
       "  0.7003348274898363,\n",
       "  0.7003619641753942,\n",
       "  0.7003748655516543,\n",
       "  0.7003856412669878,\n",
       "  0.7003951919433039,\n",
       "  0.700415350218758,\n",
       "  0.7004363418067202,\n",
       "  0.7004561950306231,\n",
       "  0.7004963611597009,\n",
       "  0.700516629095354,\n",
       "  0.7005315805492779,\n",
       "  0.7005430371743678,\n",
       "  0.7005505413252869,\n",
       "  0.7005575872681958,\n",
       "  0.7005972865118334,\n",
       "  0.7006069209176303,\n",
       "  0.7006172814564108,\n",
       "  0.7006307572799093,\n",
       "  0.7006540089653731,\n",
       "  0.7006652339475229,\n",
       "  0.7006763062792439,\n",
       "  0.7006895517982555,\n",
       "  0.7006954138613772,\n",
       "  0.7007211847484048,\n",
       "  0.7007454169032031,\n",
       "  0.700789944944785,\n",
       "  0.7008011970141911,\n",
       "  0.7008079785020311,\n",
       "  0.7008212431403762,\n",
       "  0.7008321311257367,\n",
       "  0.7008630321553758,\n",
       "  0.700875477487493,\n",
       "  0.7008900977948403,\n",
       "  0.7009040544718849,\n",
       "  0.70093507851047,\n",
       "  0.7009435575583574,\n",
       "  0.7009562658593587,\n",
       "  0.7009848383189161,\n",
       "  0.7010036208735486,\n",
       "  0.7010146126965306,\n",
       "  0.7010203318722336,\n",
       "  0.70103089272946,\n",
       "  0.7010408745833173,\n",
       "  0.7010502420678781,\n",
       "  0.7010617957462514,\n",
       "  0.701070890608019,\n",
       "  0.7010949953389419,\n",
       "  0.7011098201938225,\n",
       "  0.7011190498761332,\n",
       "  0.7011269601977724,\n",
       "  0.7011402142906088,\n",
       "  0.7011471975538949,\n",
       "  0.7011566244452139,\n",
       "  0.7011729102269616,\n",
       "  0.7011809780178467,\n",
       "  0.7012073522824371,\n",
       "  0.701218589617617,\n",
       "  0.7012325898969632,\n",
       "  0.7012577551108331,\n",
       "  0.7012710908193643,\n",
       "  0.7012789769581508,\n",
       "  0.7012960358176317,\n",
       "  0.7013061496186266,\n",
       "  0.7013228225780108,\n",
       "  0.7013280682598784,\n",
       "  0.7013446590765066,\n",
       "  0.7013632034864591,\n",
       "  0.7013796639137974,\n",
       "  0.7013878701999609,\n",
       "  0.7013997861999005,\n",
       "  0.7014140194594966,\n",
       "  0.7014259030206645,\n",
       "  0.7014384065736424,\n",
       "  0.7014509924677358,\n",
       "  0.701476878930054,\n",
       "  0.701483312858257,\n",
       "  0.7015130471070147,\n",
       "  0.7015219896715618,\n",
       "  0.7015406173120847,\n",
       "  0.7015511336105451,\n",
       "  0.7015657625301196,\n",
       "  0.7015771836737598,\n",
       "  0.7015942330529347,\n",
       "  0.7016093329999008,\n",
       "  0.7016261090174595,\n",
       "  0.7016359126573355,\n",
       "  0.701647585070949,\n",
       "  0.7016639903896419,\n",
       "  0.7016714584480214,\n",
       "  0.7016831294925546,\n",
       "  0.7016946363924577,\n",
       "  0.7017006003430972,\n",
       "  0.7017114897810837,\n",
       "  0.7017236054146627,\n",
       "  0.7017292485355909,\n",
       "  0.7017558841182246,\n",
       "  0.7017712586613242,\n",
       "  0.7017885844826456,\n",
       "  0.7018053022635463,\n",
       "  0.701813090675207,\n",
       "  0.701823398603289,\n",
       "  0.7018285715053537,\n",
       "  0.7018426279046706,\n",
       "  0.7018513814896975,\n",
       "  0.7018616008971174,\n",
       "  0.7018797112654687,\n",
       "  0.7018871690421546,\n",
       "  0.7019042805333742,\n",
       "  0.7019099275051524,\n",
       "  0.7019209829745087,\n",
       "  0.701930459475051,\n",
       "  0.7019396193375155,\n",
       "  0.7019461651677738,\n",
       "  0.7019548083095964,\n",
       "  0.7019593547549078,\n",
       "  0.7019673439725095,\n",
       "  0.7020045817982455,\n",
       "  0.7020122590949462,\n",
       "  0.7020194425435844,\n",
       "  0.7020279606386352,\n",
       "  0.702047943008283,\n",
       "  0.7020537731473149,\n",
       "  0.7020601912782135,\n",
       "  0.7020860796204419,\n",
       "  0.7020924586947018,\n",
       "  0.7021135323898867,\n",
       "  0.702127060033151,\n",
       "  0.7021401360636524,\n",
       "  0.7021576318775763,\n",
       "  0.7021796519176209,\n",
       "  0.7021824329854506,\n",
       "  0.702188048652908,\n",
       "  0.7021999256826129,\n",
       "  0.7022141010449646,\n",
       "  0.702237128731196,\n",
       "  0.7022454363896591,\n",
       "  0.702251189313106,\n",
       "  0.7022663237228167,\n",
       "  0.7022792422399685,\n",
       "  0.7022895124948287,\n",
       "  0.7023018943395685,\n",
       "  0.7023100676224622,\n",
       "  0.7023196778314267,\n",
       "  0.7023301801991192,\n",
       "  0.7023379620360436,\n",
       "  0.7023468887349286,\n",
       "  0.7023575881891806,\n",
       "  0.7023676253535251,\n",
       "  0.7023969647746385,\n",
       "  0.7024085378288215,\n",
       "  0.7024263880701319,\n",
       "  0.7024328026121642,\n",
       "  0.702452545963586,\n",
       "  0.7024650456348162,\n",
       "  0.702471192311818,\n",
       "  0.7024791604428574,\n",
       "  0.7024880358996214,\n",
       "  0.7024949022010262,\n",
       "  0.7025030329344508,\n",
       "  0.7025163165497853,\n",
       "  0.7025209797537754,\n",
       "  0.7025329427939756,\n",
       "  0.7025403218523039,\n",
       "  0.702557394167017,\n",
       "  0.70256894363934,\n",
       "  0.7025883001611116,\n",
       "  0.702597117912862,\n",
       "  0.7026230162495555,\n",
       "  0.7026290606450433,\n",
       "  0.7026380596089238,\n",
       "  0.7026411597463795,\n",
       "  0.7026464491044657,\n",
       "  0.7026614669120058,\n",
       "  0.7026709362269962,\n",
       "  0.7026787208184802,\n",
       "  0.7026920839311835,\n",
       "  0.7026974536987988,\n",
       "  0.7027111579842383,\n",
       "  0.7027377702593195,\n",
       "  0.7027420344042942,\n",
       "  0.7027495313619702,\n",
       "  0.7027568448256115,\n",
       "  0.7027699989406128,\n",
       "  0.7027758404789098,\n",
       "  0.7027792647573143,\n",
       "  0.7027846738535906,\n",
       "  0.7027911399339399,\n",
       "  0.7027996747479891,\n",
       "  0.7028078832615856,\n",
       "  0.7028214014879524,\n",
       "  0.7028264445219214,\n",
       "  0.7028384376412449,\n",
       "  0.7028491199749077,\n",
       "  0.7028609299734162,\n",
       "  0.702877394263575,\n",
       "  0.7028824833884882,\n",
       "  0.7028896312927242,\n",
       "  0.7028985815786267,\n",
       "  0.702913843116599,\n",
       "  0.7029195070986073,\n",
       "  0.7029331617546694,\n",
       "  0.7029432696636699,\n",
       "  0.7029524756931131,\n",
       "  0.7029582568970625,\n",
       "  0.7029693113654002,\n",
       "  0.7029765126203222,\n",
       "  0.7029995423929508,\n",
       "  0.7030239034393443,\n",
       "  0.7030326636687217,\n",
       "  0.7030396781486457,\n",
       "  0.7030531857365694,\n",
       "  0.7030632458665523,\n",
       "  0.7030872782327214,\n",
       "  0.703094936098905,\n",
       "  0.7031049869032501,\n",
       "  0.7031172152245627,\n",
       "  0.7031215539543391,\n",
       "  0.7031333552338271,\n",
       "  0.70314016574536,\n",
       "  0.703147790375171,\n",
       "  0.7031598315942436,\n",
       "  0.703169435149405,\n",
       "  0.70317309371526,\n",
       "  0.7031804116365227,\n",
       "  0.7031838106282998,\n",
       "  0.7031898909814704,\n",
       "  0.7031982462069483,\n",
       "  0.7032014994891226,\n",
       "  0.7032073596397742,\n",
       "  0.7032145898613787,\n",
       "  0.7032210889074587,\n",
       "  0.7032265610256858,\n",
       "  0.7032411023023667,\n",
       "  0.7032488995119053,\n",
       "  0.7032754903349495,\n",
       "  0.703281143421389,\n",
       "  0.7032926337921274,\n",
       "  0.703297391906155,\n",
       "  0.7033051432256889,\n",
       "  0.7033139588778746,\n",
       "  0.7033193935347315,\n",
       "  0.7033282929139416,\n",
       "  0.70334443697787,\n",
       "  0.7033604653757646,\n",
       "  0.7033700411651281,\n",
       "  0.703380791318725,\n",
       "  0.7033896252706234,\n",
       "  0.7034135151334414,\n",
       "  0.7034252256009363,\n",
       "  0.7034291414278168,\n",
       "  0.7034399101585741,\n",
       "  0.7034457530521395,\n",
       "  0.703456439242145,\n",
       "  0.7034640665696631,\n",
       "  0.7034731665133306,\n",
       "  0.7034797607848432,\n",
       "  0.7034924947108401,\n",
       "  0.7035023361515882,\n",
       "  0.7035118205707173,\n",
       "  0.7035231347563133,\n",
       "  0.7035313624085751,\n",
       "  0.7035384177967372,\n",
       "  0.7035436377431638,\n",
       "  0.7035500920444453,\n",
       "  0.7035567493310207,\n",
       "  0.7035633508482516,\n",
       "  0.7035749922027842,\n",
       "  0.7035889529938159,\n",
       "  0.7035988219955166,\n",
       "  0.7036137734274618,\n",
       "  0.7036247869348845,\n",
       "  0.7036311074663301,\n",
       "  0.7036464833172659,\n",
       "  0.7036538985565378,\n",
       "  0.7036614045244249,\n",
       "  0.7036711431606847,\n",
       "  0.7036765843257969,\n",
       "  0.7036973923159735,\n",
       "  0.7037014638485166,\n",
       "  0.7037090392113337,\n",
       "  0.7037238853686096,\n",
       "  0.7037266001567709,\n",
       "  0.7037388312657114,\n",
       "  0.7037454807045842,\n",
       "  0.7037730845934509,\n",
       "  0.7038019438094851,\n",
       "  0.7038104888989408,\n",
       "  0.7038175039383617,\n",
       "  0.7038299014607499,\n",
       "  0.7038442385371457,\n",
       "  0.7038539047118982,\n",
       "  0.7038596547025703,\n",
       "  0.7038685911611496,\n",
       "  0.7038806702228725,\n",
       "  0.7038870221436082,\n",
       "  0.7038950148146753,\n",
       "  0.7039043113046084,\n",
       "  0.7039123072366754,\n",
       "  0.7039574992640426,\n",
       "  0.7039697115850303,\n",
       "  0.7039806425824926,\n",
       "  0.7040010766105316,\n",
       "  0.7040087642857795,\n",
       "  0.7040180500088974,\n",
       "  0.7040257772401937,\n",
       "  0.7040271499861014,\n",
       "  0.7040320021868992,\n",
       "  0.7040519008353292,\n",
       "  0.7040610206244104,\n",
       "  0.7040648050044118,\n",
       "  0.7040677598099832,\n",
       "  0.7040736671948767,\n",
       "  0.7040791955141478,\n",
       "  0.7040838454906603,\n",
       "  0.7040898591138248,\n",
       "  0.7041237834846724,\n",
       "  0.7041409754601315,\n",
       "  0.7041464652267306,\n",
       "  0.7041533653521467,\n",
       "  0.7041632538620721,\n",
       "  0.7041905451945363,\n",
       "  0.7041999756051296,\n",
       "  0.7042138859326869,\n",
       "  0.7042349215674153,\n",
       "  0.7042410724004181,\n",
       "  0.7042472761143775,\n",
       "  0.7042546880855083,\n",
       "  0.7042588938059057,\n",
       "  0.7042664040594793,\n",
       "  0.7042744404258864,\n",
       "  0.7042835153378232,\n",
       "  0.7042896048935957,\n",
       "  0.7042967027181789,\n",
       "  0.7043268437702462,\n",
       "  0.7043345283061119,\n",
       "  0.7043379538157689,\n",
       "  0.7043508293379904,\n",
       "  0.7043656778116597,\n",
       "  0.7043718772584037,\n",
       "  0.7043751129205117,\n",
       "  0.7043846999410452,\n",
       "  0.7043970186214654,\n",
       "  0.7044036898167524,\n",
       "  0.7044180690997522,\n",
       "  0.7044219196269021,\n",
       "  0.704430622912696,\n",
       "  0.7044382847888494,\n",
       "  0.7044473859406147,\n",
       "  0.7044601674527081,\n",
       "  0.7044654088302105,\n",
       "  0.7044743428276382,\n",
       "  0.7044825677910245,\n",
       "  0.704489924191488,\n",
       "  0.7045003516883569,\n",
       "  0.7045079928981463,\n",
       "  0.7045142207837625,\n",
       "  0.7045216839491034,\n",
       "  0.7045401215115881,\n",
       "  0.7045460886929932,\n",
       "  0.7045507489338388,\n",
       "  0.7045594697335646,\n",
       "  0.7045654973399649,\n",
       "  0.7045707748349853,\n",
       "  0.704581024468391,\n",
       "  0.7045873532338565,\n",
       "  0.7045927488366154,\n",
       "  0.7046047906872174,\n",
       "  0.7046140551542259,\n",
       "  0.7046211459570021,\n",
       "  0.7046242117806621,\n",
       "  0.7046300476197125,\n",
       "  0.7046397732850943,\n",
       "  0.7046438345246261,\n",
       "  0.704654323176238,\n",
       "  0.7046718081375897,\n",
       "  0.7046784189037818,\n",
       "  0.7046977590337867,\n",
       "  0.7047068931661054,\n",
       "  0.7047160841709376,\n",
       "  0.7047285213998934,\n",
       "  0.7047416614734882,\n",
       "  0.7047467836117199,\n",
       "  0.7047583177106697,\n",
       "  0.7047656997317215,\n",
       "  0.7047752831995835,\n",
       "  ...],\n",
       " 'auc-stdv': [0.00010116967523407818,\n",
       "  8.249704267786946e-05,\n",
       "  0.00023004758081113987,\n",
       "  0.0002866374669493299,\n",
       "  0.00027658985267397314,\n",
       "  0.00023742951732145425,\n",
       "  0.0001974664916239585,\n",
       "  0.00023638562734723634,\n",
       "  0.00021315601050878638,\n",
       "  0.0002289234646135239,\n",
       "  0.00031084409428245945,\n",
       "  0.00027535742812910915,\n",
       "  0.00022730768676041354,\n",
       "  0.0002017996574027483,\n",
       "  0.0002924782875977275,\n",
       "  0.00023389111420777384,\n",
       "  0.00022659829306935703,\n",
       "  0.00023256592081695302,\n",
       "  0.00022341124254338847,\n",
       "  0.0002332521222022905,\n",
       "  0.00021447720399936775,\n",
       "  0.0002031698068586921,\n",
       "  0.00020567977137228994,\n",
       "  0.00013655352777118738,\n",
       "  0.0001249230904316996,\n",
       "  0.00014685291566475192,\n",
       "  0.00013911345699126634,\n",
       "  0.00010250337628064407,\n",
       "  9.489888949403148e-05,\n",
       "  0.00010787549797911609,\n",
       "  8.299923368483707e-05,\n",
       "  0.00010822740991550754,\n",
       "  0.00018315871344907014,\n",
       "  0.0002284276560766122,\n",
       "  0.0002752138891780222,\n",
       "  0.00027292400494490236,\n",
       "  0.00033642776704662287,\n",
       "  0.0003558668392964933,\n",
       "  0.0003478159671973006,\n",
       "  0.000338202337974222,\n",
       "  0.0003361945036292437,\n",
       "  0.00033890309357001685,\n",
       "  0.0003558054999655877,\n",
       "  0.00041866213206523905,\n",
       "  0.00041228891263709027,\n",
       "  0.0004921113083771529,\n",
       "  0.0004627186659561423,\n",
       "  0.0004910324292465991,\n",
       "  0.00039267906274427966,\n",
       "  0.00039183283631849747,\n",
       "  0.0003593087955013628,\n",
       "  0.00042382206927133706,\n",
       "  0.00043948311507646795,\n",
       "  0.00043523646880686586,\n",
       "  0.0004152860559658513,\n",
       "  0.0004483786615575958,\n",
       "  0.0004409336649963056,\n",
       "  0.0004165584293866919,\n",
       "  0.00045492758227477754,\n",
       "  0.0004995520245260006,\n",
       "  0.0005245974194664494,\n",
       "  0.0004736541413232257,\n",
       "  0.0004523373417527116,\n",
       "  0.0004540896076846627,\n",
       "  0.00044010602196161504,\n",
       "  0.0004462791954018026,\n",
       "  0.0005040612912462403,\n",
       "  0.0005393798070753148,\n",
       "  0.0005848540581424562,\n",
       "  0.0005816830654178333,\n",
       "  0.000517625724576125,\n",
       "  0.0005300480946973564,\n",
       "  0.000512533671881662,\n",
       "  0.00046679893191078334,\n",
       "  0.00045730444918356384,\n",
       "  0.00047938429317806767,\n",
       "  0.0004797028801508968,\n",
       "  0.00047545470209337063,\n",
       "  0.0004797803196001879,\n",
       "  0.00049433388225902,\n",
       "  0.00048313091959042997,\n",
       "  0.0004717389913202266,\n",
       "  0.00045816146670148215,\n",
       "  0.00044640863243272807,\n",
       "  0.00040401668394920537,\n",
       "  0.00043179609611939274,\n",
       "  0.0004523935164213447,\n",
       "  0.00045657839335954884,\n",
       "  0.0004832529515286228,\n",
       "  0.00048256042880347134,\n",
       "  0.0004532751708219757,\n",
       "  0.0004647186363277234,\n",
       "  0.0004768736493306981,\n",
       "  0.0004335528583809496,\n",
       "  0.00039395387448524897,\n",
       "  0.0003863104251374748,\n",
       "  0.00038140998290959625,\n",
       "  0.0003907764549363199,\n",
       "  0.0003649139175695498,\n",
       "  0.0003701554543412618,\n",
       "  0.00037823149522402827,\n",
       "  0.0003208640906079255,\n",
       "  0.0003501228666715167,\n",
       "  0.00034914330371429487,\n",
       "  0.00039011291629697094,\n",
       "  0.0004524108412002775,\n",
       "  0.00048129779256800623,\n",
       "  0.0005308078799592053,\n",
       "  0.0005661679307875891,\n",
       "  0.000550597263333231,\n",
       "  0.0005383574171041267,\n",
       "  0.0004990057810291209,\n",
       "  0.0004860229672751112,\n",
       "  0.0005056924850105438,\n",
       "  0.000493503670527631,\n",
       "  0.0004735707923478334,\n",
       "  0.0004581178521293077,\n",
       "  0.000470173455532633,\n",
       "  0.00045760889810629016,\n",
       "  0.0004813658046555577,\n",
       "  0.00048475474258902445,\n",
       "  0.00046258332563686087,\n",
       "  0.00047616329314155664,\n",
       "  0.00043061479162359145,\n",
       "  0.0004447059613496608,\n",
       "  0.00044928620727204986,\n",
       "  0.0004872953485399136,\n",
       "  0.0004984587993985335,\n",
       "  0.00047118421283658005,\n",
       "  0.0004268545234685957,\n",
       "  0.00042716289825120825,\n",
       "  0.0004171816528560727,\n",
       "  0.0004197459622600833,\n",
       "  0.0004055367729502059,\n",
       "  0.00040205739217714383,\n",
       "  0.0004461527747999232,\n",
       "  0.0004160559269040741,\n",
       "  0.00041365084096159485,\n",
       "  0.0003746738339327816,\n",
       "  0.00041025846783561285,\n",
       "  0.00037670918135966684,\n",
       "  0.00038723845932345165,\n",
       "  0.00038979198179448995,\n",
       "  0.0003766959184678899,\n",
       "  0.0003607173682354797,\n",
       "  0.0003574133190140518,\n",
       "  0.00033025877326299963,\n",
       "  0.0003310939371205943,\n",
       "  0.00034124074110372237,\n",
       "  0.00034358400347842043,\n",
       "  0.0003050701226394127,\n",
       "  0.00030605880211452324,\n",
       "  0.00028600915975729153,\n",
       "  0.0002626457115622285,\n",
       "  0.00025319808922068184,\n",
       "  0.0002637321599950298,\n",
       "  0.00028006917848608933,\n",
       "  0.00029405556605341474,\n",
       "  0.000267726125909912,\n",
       "  0.00022788689537824635,\n",
       "  0.0002175570620001607,\n",
       "  0.0002186196799242011,\n",
       "  0.0002301620613528165,\n",
       "  0.0001886081333369017,\n",
       "  0.00014436890723413994,\n",
       "  0.00017287796452223193,\n",
       "  0.00016845222979904433,\n",
       "  0.00018371611958263194,\n",
       "  0.00019481699381258754,\n",
       "  0.0002044012848374567,\n",
       "  0.0002281890762775316,\n",
       "  0.00023417635588123922,\n",
       "  0.00021479464745433722,\n",
       "  0.0002452055360162289,\n",
       "  0.00021426453812434723,\n",
       "  0.00020399075039004345,\n",
       "  0.0001982222910763382,\n",
       "  0.00019363198600720578,\n",
       "  0.00019029669874802874,\n",
       "  0.00018745273961901318,\n",
       "  0.00018544151508601147,\n",
       "  0.0001923061931530163,\n",
       "  0.00018110119086807403,\n",
       "  0.00019390749916113465,\n",
       "  0.0002006759939029806,\n",
       "  0.00019614971235102358,\n",
       "  0.00018259291098881477,\n",
       "  0.00021046680883201127,\n",
       "  0.0002020725202021579,\n",
       "  0.00019000838697247154,\n",
       "  0.00014874208134983263,\n",
       "  0.00014781275323482788,\n",
       "  0.00016281291296836584,\n",
       "  0.00017191638306012256,\n",
       "  0.00018331621904725447,\n",
       "  0.00019047718998134062,\n",
       "  0.00019886162730971366,\n",
       "  0.0002027347323587774,\n",
       "  0.0001958750791597416,\n",
       "  0.00022760346318887955,\n",
       "  0.00019328007140710906,\n",
       "  0.00022417451617955325,\n",
       "  0.00022685953187503083,\n",
       "  0.0002255858519382278,\n",
       "  0.00022548647099700824,\n",
       "  0.00023662110898727858,\n",
       "  0.0002334743651284795,\n",
       "  0.00023820874539420578,\n",
       "  0.00022160225764256727,\n",
       "  0.0002461686650166342,\n",
       "  0.00028641255399338827,\n",
       "  0.0003018761348955515,\n",
       "  0.00029894236820209036,\n",
       "  0.00029600124736967265,\n",
       "  0.0003028066267653317,\n",
       "  0.00030584891569744436,\n",
       "  0.0003025180725152981,\n",
       "  0.00033766939783005907,\n",
       "  0.0003106865198420502,\n",
       "  0.0002703993892072317,\n",
       "  0.00027940916266393077,\n",
       "  0.0002831520642378248,\n",
       "  0.00027933007057177536,\n",
       "  0.0002867651386234159,\n",
       "  0.0002964653347009527,\n",
       "  0.0003226240539611517,\n",
       "  0.0003125840772696732,\n",
       "  0.0003374509415065088,\n",
       "  0.0003526194871442951,\n",
       "  0.0003562015278138532,\n",
       "  0.0003399990463882953,\n",
       "  0.00031783966240556724,\n",
       "  0.00029695499801171926,\n",
       "  0.0002808198229316911,\n",
       "  0.000306065676999341,\n",
       "  0.00029475472587559703,\n",
       "  0.00031381818387185307,\n",
       "  0.0003090684397230358,\n",
       "  0.00030386482620701404,\n",
       "  0.00030722970266283916,\n",
       "  0.0002935602126043528,\n",
       "  0.00031781206884286354,\n",
       "  0.00031840079690901793,\n",
       "  0.0003277162248292968,\n",
       "  0.0003303025066120772,\n",
       "  0.00029940891281959273,\n",
       "  0.0003025298127879558,\n",
       "  0.00031764818496956394,\n",
       "  0.00031947622251419974,\n",
       "  0.0003434337486621944,\n",
       "  0.00034152598494816737,\n",
       "  0.0003498218446829151,\n",
       "  0.0003367874701612451,\n",
       "  0.00034374550323450497,\n",
       "  0.0003369174839276704,\n",
       "  0.00032018155043950827,\n",
       "  0.00033462152797459184,\n",
       "  0.0003493036675654885,\n",
       "  0.0003492769663509528,\n",
       "  0.00035022864857971436,\n",
       "  0.0003860848086980607,\n",
       "  0.0004075304764298501,\n",
       "  0.0004112855828909583,\n",
       "  0.00043049388564722667,\n",
       "  0.00041830440098770364,\n",
       "  0.0004106281549081096,\n",
       "  0.00042784113139809584,\n",
       "  0.0004358509452459364,\n",
       "  0.00044819246440163025,\n",
       "  0.0004643243802302824,\n",
       "  0.0004923585053256213,\n",
       "  0.0004824537378729949,\n",
       "  0.0004986620196241027,\n",
       "  0.0005019120108060503,\n",
       "  0.0004878891922659549,\n",
       "  0.0004844725829026555,\n",
       "  0.0005198987878244943,\n",
       "  0.0005253240283911815,\n",
       "  0.0005240765406651478,\n",
       "  0.0005297381350387765,\n",
       "  0.0005409256104668625,\n",
       "  0.0005464783371256135,\n",
       "  0.0005481236022826773,\n",
       "  0.0005468657523508635,\n",
       "  0.0005591354137772008,\n",
       "  0.0005681005492232557,\n",
       "  0.0005752043350037404,\n",
       "  0.0005786746754224525,\n",
       "  0.0005639514126020059,\n",
       "  0.0005463292875806272,\n",
       "  0.0005521061728397946,\n",
       "  0.0005540135078152555,\n",
       "  0.0005574170694819337,\n",
       "  0.0005531244531967095,\n",
       "  0.0005478179903563262,\n",
       "  0.0005326155119912982,\n",
       "  0.0005372623130145515,\n",
       "  0.0005517889401497703,\n",
       "  0.0005242400445785254,\n",
       "  0.0005268221871117871,\n",
       "  0.0005364237174850224,\n",
       "  0.0005219656802935007,\n",
       "  0.0005253501520568312,\n",
       "  0.0005213027870673429,\n",
       "  0.0005251866842525352,\n",
       "  0.0005780943363531296,\n",
       "  0.0005573945228671824,\n",
       "  0.0005499575134462056,\n",
       "  0.0005651192128790725,\n",
       "  0.0005449833567058223,\n",
       "  0.0005358286555723873,\n",
       "  0.0005312300679305327,\n",
       "  0.0005285606119643707,\n",
       "  0.0005213608814950336,\n",
       "  0.0005214332697431699,\n",
       "  0.0005180152244421203,\n",
       "  0.0005113343359521014,\n",
       "  0.0005258653001521546,\n",
       "  0.0005298576876978389,\n",
       "  0.0005289853550076051,\n",
       "  0.0005326879597173191,\n",
       "  0.0005356687125388402,\n",
       "  0.0005009009374973592,\n",
       "  0.0005108247303292419,\n",
       "  0.0004936124529284888,\n",
       "  0.0004969396835112264,\n",
       "  0.000489819608276412,\n",
       "  0.0004819557189762781,\n",
       "  0.0004915322496510254,\n",
       "  0.0004941347078289939,\n",
       "  0.0005146781614580424,\n",
       "  0.0005218173675176457,\n",
       "  0.0005104401988839205,\n",
       "  0.0005098443755279915,\n",
       "  0.0005097475888443924,\n",
       "  0.0004879623543129949,\n",
       "  0.0004925478256452485,\n",
       "  0.000500991760526005,\n",
       "  0.0004897236628117837,\n",
       "  0.0004958502881031269,\n",
       "  0.0004868408530426827,\n",
       "  0.0005020020588696035,\n",
       "  0.0004939559452330738,\n",
       "  0.0005058537873368898,\n",
       "  0.000505284104391127,\n",
       "  0.0005028113725774435,\n",
       "  0.0005287012597932937,\n",
       "  0.0005265828822706901,\n",
       "  0.0005118570873862197,\n",
       "  0.0005200710079203984,\n",
       "  0.0004984827375520671,\n",
       "  0.000505779894883837,\n",
       "  0.0005008722611431837,\n",
       "  0.0005041039213646327,\n",
       "  0.0005037378034637354,\n",
       "  0.000496464673060928,\n",
       "  0.0004917966490146406,\n",
       "  0.0004870450859458625,\n",
       "  0.00047741840266679944,\n",
       "  0.00047543652316267456,\n",
       "  0.0004719513522762388,\n",
       "  0.0004706173353994615,\n",
       "  0.0004750673793723443,\n",
       "  0.00046109719744384534,\n",
       "  0.000455118729016033,\n",
       "  0.0004570334182093275,\n",
       "  0.0004582275888962518,\n",
       "  0.0004459526729813233,\n",
       "  0.0004427875967513672,\n",
       "  0.00042511227599090386,\n",
       "  0.0004284916620761331,\n",
       "  0.00041094358181742994,\n",
       "  0.00040705008844544736,\n",
       "  0.00040282278870405085,\n",
       "  0.00041140429304569354,\n",
       "  0.0004112439330782932,\n",
       "  0.00040509831292024744,\n",
       "  0.00040040870905134313,\n",
       "  0.0004019218190438549,\n",
       "  0.00040219297268432794,\n",
       "  0.0003932012642527848,\n",
       "  0.00039740982438396137,\n",
       "  0.00040302389942671726,\n",
       "  0.000397914984639978,\n",
       "  0.0003957837755765027,\n",
       "  0.00039224574658175827,\n",
       "  0.0004043247099861849,\n",
       "  0.00042635019297988975,\n",
       "  0.0004174331614175111,\n",
       "  0.00042157484514314473,\n",
       "  0.00043447726988917184,\n",
       "  0.0004385976488165776,\n",
       "  0.0004370336715783321,\n",
       "  0.00043944608053492624,\n",
       "  0.00043375476014534273,\n",
       "  0.00043026595880385975,\n",
       "  0.0004317777705267445,\n",
       "  0.0004291776348891981,\n",
       "  0.00042238169554777665,\n",
       "  0.0004140122425824585,\n",
       "  0.0004151919718915783,\n",
       "  0.00041533915646880295,\n",
       "  0.00041060397042533373,\n",
       "  0.0004106272211339311,\n",
       "  0.0004094289801790568,\n",
       "  0.0004183596941302192,\n",
       "  0.00042903120352791145,\n",
       "  0.0004230786000378353,\n",
       "  0.00043644332692087715,\n",
       "  0.00043351525968225545,\n",
       "  0.0004265728495693871,\n",
       "  0.00042633779292482584,\n",
       "  0.000429940728807893,\n",
       "  0.00042370059143050537,\n",
       "  0.00040468173643220097,\n",
       "  0.0004125594116881389,\n",
       "  0.00040723608536482106,\n",
       "  0.00040543260830634025,\n",
       "  0.00040023434462931805,\n",
       "  0.0004081105271890044,\n",
       "  0.0004087538215200835,\n",
       "  0.00040604162947151235,\n",
       "  0.00040935421051815316,\n",
       "  0.0004194692740941675,\n",
       "  0.0004158876513374168,\n",
       "  0.00042189019837245867,\n",
       "  0.00042233417360687417,\n",
       "  0.00041892026279575893,\n",
       "  0.00043201002766296017,\n",
       "  0.00043337095651413155,\n",
       "  0.0004338951428286301,\n",
       "  0.00043057849772451794,\n",
       "  0.00042408184495413604,\n",
       "  0.00043566832375773536,\n",
       "  0.0004356970031119756,\n",
       "  0.00043599874895033946,\n",
       "  0.0004249871380146062,\n",
       "  0.0004300974974075266,\n",
       "  0.00042762127928665234,\n",
       "  0.0004477622913321827,\n",
       "  0.00044838327462103176,\n",
       "  0.000447606462689222,\n",
       "  0.0004456549858895195,\n",
       "  0.00043999422884691774,\n",
       "  0.00041431058501365496,\n",
       "  0.0004068717797883846,\n",
       "  0.0004100741755710124,\n",
       "  0.00041482850559110236,\n",
       "  0.00040413693895767277,\n",
       "  0.0003959037434658642,\n",
       "  0.0003960396051719673,\n",
       "  0.000393735489738349,\n",
       "  0.0003936919533935634,\n",
       "  0.0003909015970215652,\n",
       "  0.0003783356978443379,\n",
       "  0.00037967227522790205,\n",
       "  0.0003711755940262097,\n",
       "  0.0003725091609578705,\n",
       "  0.00037304969417483743,\n",
       "  0.00036717818454493013,\n",
       "  0.00036701304366105564,\n",
       "  0.0003713600063709075,\n",
       "  0.0003675466130863813,\n",
       "  0.0003701259708794087,\n",
       "  0.00036676663947838694,\n",
       "  0.0003582633257364486,\n",
       "  0.0003311106196629969,\n",
       "  0.0003375421884226952,\n",
       "  0.00032949580251560894,\n",
       "  0.0003301760583850214,\n",
       "  0.0003435816396994199,\n",
       "  0.00034862362269482794,\n",
       "  0.00035237678600889935,\n",
       "  0.0003502033472945942,\n",
       "  0.0003525563913966021,\n",
       "  0.0003622066310856841,\n",
       "  0.000373186238916327,\n",
       "  0.00039192328044649314,\n",
       "  0.00039367027503346604,\n",
       "  0.00039141657931205634,\n",
       "  0.0003945480462370393,\n",
       "  0.0003929895125545181,\n",
       "  0.0003974183261084014,\n",
       "  0.00039676057538058364,\n",
       "  0.0003966774050874436,\n",
       "  0.0003956898627163336,\n",
       "  0.0003987450416566311,\n",
       "  0.0004041766573446622,\n",
       "  0.00039954788374449435,\n",
       "  0.0003954270573632725,\n",
       "  0.0003998317831894416,\n",
       "  0.0004032333025181295,\n",
       "  0.00041484665158135687,\n",
       "  0.0004481202509569434,\n",
       "  0.0004479800421666459,\n",
       "  0.00044942986217615116,\n",
       "  0.00044336244909514886,\n",
       "  0.0004276746452519009,\n",
       "  0.0004399540130473711,\n",
       "  0.0004400001268204076,\n",
       "  0.000454958546625272,\n",
       "  0.00044816288740527426,\n",
       "  0.0004591612920013066,\n",
       "  0.00044885798600012317,\n",
       "  0.00044779032351667514,\n",
       "  0.0004408600947040226,\n",
       "  0.00043847234897863866,\n",
       "  0.0004412550792835778,\n",
       "  0.00041378334008613577,\n",
       "  0.0004230729634941631,\n",
       "  0.00042020635785087427,\n",
       "  0.000418524506207997,\n",
       "  0.00041983533683088775,\n",
       "  0.0004157731412678314,\n",
       "  0.0004124806757874335,\n",
       "  0.0004025765932068539,\n",
       "  0.0004218020265986638,\n",
       "  0.00041620795852673567,\n",
       "  0.00042167151119707884,\n",
       "  0.0004191252899219951,\n",
       "  0.000417270553872137,\n",
       "  0.0003940137429966734,\n",
       "  0.0003923560736393793,\n",
       "  0.00038611710163341366,\n",
       "  0.0003892269555221676,\n",
       "  0.0003940907364759668,\n",
       "  0.0003943711638536537,\n",
       "  0.0003919813462685314,\n",
       "  0.00039872355062714297,\n",
       "  0.00040095183999806425,\n",
       "  0.0004014346147623123,\n",
       "  0.00040972780426525734,\n",
       "  0.00040980191854849806,\n",
       "  0.00041190691438417875,\n",
       "  0.00041012157259704763,\n",
       "  0.0004051307133504114,\n",
       "  0.00040850145892630703,\n",
       "  0.0004068402254069152,\n",
       "  0.0004206343046687822,\n",
       "  0.0004267541171755514,\n",
       "  0.00040792142099365335,\n",
       "  0.0004007724637196852,\n",
       "  0.0003944808601551294,\n",
       "  0.0003969405721275789,\n",
       "  0.0004191651026733282,\n",
       "  0.0004264454697187195,\n",
       "  0.000417045688150201,\n",
       "  0.000419151341984669,\n",
       "  0.0004133342670381582,\n",
       "  0.0004008928036888957,\n",
       "  0.0004032427512912706,\n",
       "  0.00040077170479989213,\n",
       "  0.0003971549490217092,\n",
       "  0.0003927031650999755,\n",
       "  0.00038822805404945036,\n",
       "  0.00039880656165756684,\n",
       "  0.00040062629333885555,\n",
       "  0.000400582182235924,\n",
       "  0.00039726153777088467,\n",
       "  0.0004010551012685225,\n",
       "  0.00040862232910825213,\n",
       "  0.0003870585000645084,\n",
       "  0.0004031961835487958,\n",
       "  0.0004021461371193774,\n",
       "  0.0004103283298675606,\n",
       "  0.00041370336695848326,\n",
       "  0.00041680844572137307,\n",
       "  0.00042094434484192544,\n",
       "  0.0004210342045201933,\n",
       "  0.00042309045946922557,\n",
       "  0.0004147973795448445,\n",
       "  0.00041678817415641533,\n",
       "  0.000416037868467055,\n",
       "  0.0004125552981668571,\n",
       "  0.00041004580765639293,\n",
       "  0.000404069997680692,\n",
       "  0.00039209482486439256,\n",
       "  0.0003870666589214385,\n",
       "  0.0003879216150077577,\n",
       "  0.0003929748532388542,\n",
       "  0.00039845712073594984,\n",
       "  0.00040157620178170073,\n",
       "  0.00039729545842605545,\n",
       "  0.0003980695290851065,\n",
       "  0.00039216967494614844,\n",
       "  0.00039355522778213123,\n",
       "  0.00038989128931981047,\n",
       "  0.00041783340705025943,\n",
       "  0.0004219727364768713,\n",
       "  0.0004251715981431219,\n",
       "  0.0004171565306616028,\n",
       "  0.0004176934104658556,\n",
       "  0.0004161679059276659,\n",
       "  0.00040979777659729845,\n",
       "  0.00041137864978909114,\n",
       "  0.0004138958636078757,\n",
       "  0.00041035407987414255,\n",
       "  0.000415930219772,\n",
       "  0.0004247941517267373,\n",
       "  0.0004312103941133593,\n",
       "  0.0004252166304504126,\n",
       "  0.00040257343239345154,\n",
       "  0.0003942048766720323,\n",
       "  0.000389066614455932,\n",
       "  0.0004012657271679816,\n",
       "  0.0003982005836637587,\n",
       "  0.0003948575027126648,\n",
       "  0.0003971598861448914,\n",
       "  0.0004011280010205576,\n",
       "  0.0003990172716148623,\n",
       "  0.0004059297121892311,\n",
       "  0.0004064082027830932,\n",
       "  0.00040494641965733226,\n",
       "  0.0004021900067124322,\n",
       "  0.00040000280731254483,\n",
       "  0.0003991220694297368,\n",
       "  0.00040873728426860117,\n",
       "  0.00041473850387791313,\n",
       "  0.00041392678800222603,\n",
       "  0.0004175860114256982,\n",
       "  0.0004141471872008872,\n",
       "  0.00041444172702981177,\n",
       "  0.00040683038423582665,\n",
       "  0.000405518810107997,\n",
       "  0.00040897782541771745,\n",
       "  0.0004082645096743839,\n",
       "  0.0004058899586674369,\n",
       "  0.00040828438949712723,\n",
       "  0.0004049015539789325,\n",
       "  0.0004047817132022525,\n",
       "  0.00040378430827216525,\n",
       "  0.00040537095281399643,\n",
       "  0.00040739564856979055,\n",
       "  0.00040915699610485196,\n",
       "  0.0004086018966100114,\n",
       "  0.00040885508576868915,\n",
       "  0.0004037964062407379,\n",
       "  0.0004043414909176319,\n",
       "  0.00038703269115646815,\n",
       "  0.0003876777168428302,\n",
       "  0.0003736532843363404,\n",
       "  0.0003740082507950935,\n",
       "  0.00037597226158768914,\n",
       "  0.0003749641570249983,\n",
       "  0.0003751281835007848,\n",
       "  0.00037191909941827945,\n",
       "  0.0003690503349271665,\n",
       "  0.0003582682517202544,\n",
       "  0.0003570799333500576,\n",
       "  0.0003697329298542213,\n",
       "  0.0003711592575110863,\n",
       "  0.00037513377688686575,\n",
       "  0.0003448791662236154,\n",
       "  0.00035013351384445145,\n",
       "  0.00034211636180325636,\n",
       "  0.00034143430888859477,\n",
       "  0.00034366672030702265,\n",
       "  0.00034739905443121173,\n",
       "  0.0003414307811741649,\n",
       "  0.00034566487137869503,\n",
       "  0.0003475686073424558,\n",
       "  0.00035701177924047305,\n",
       "  0.0003536283569185792,\n",
       "  0.0003528909710868411,\n",
       "  0.0003533477951508187,\n",
       "  0.0003591968024169801,\n",
       "  0.0003618845087764577,\n",
       "  0.00036537243031007315,\n",
       "  0.0003702612172292008,\n",
       "  0.0003701842782567186,\n",
       "  0.0003928090775823778,\n",
       "  0.00039714756668654034,\n",
       "  0.00039668160153373246,\n",
       "  0.0004036725240956057,\n",
       "  0.0004021190507594945,\n",
       "  0.00040451307739332304,\n",
       "  0.0004107639364058433,\n",
       "  0.00041183552647365524,\n",
       "  0.00041075361705142594,\n",
       "  0.00041249806143145516,\n",
       "  0.00040895709006388804,\n",
       "  0.0004004347883964719,\n",
       "  0.000382955854399744,\n",
       "  0.000383698301229149,\n",
       "  0.0003833925545015183,\n",
       "  0.00038188437348087,\n",
       "  0.0003800934612866511,\n",
       "  0.0003766479527095985,\n",
       "  0.0003725247278573275,\n",
       "  0.00036651329764691634,\n",
       "  0.0003688033672515875,\n",
       "  0.0003413362210239476,\n",
       "  0.0003398123617105205,\n",
       "  0.0003392474715010587,\n",
       "  0.0003362675667439089,\n",
       "  0.0003362503546046423,\n",
       "  0.00033460077342331503,\n",
       "  0.00034325704592352716,\n",
       "  0.0003446473181033569,\n",
       "  0.00035302950617666204,\n",
       "  0.00035665298775217427,\n",
       "  0.00035841739440121403,\n",
       "  0.00035319861655858174,\n",
       "  0.0003526889408089197,\n",
       "  0.0003535759943401464,\n",
       "  0.00035094737738497093,\n",
       "  0.0003503141361120143,\n",
       "  0.000348707542250625,\n",
       "  0.00034092745718521746,\n",
       "  0.0003410019205808681,\n",
       "  0.00034337512860708715,\n",
       "  0.0003345666444005082,\n",
       "  0.0003337735660507415,\n",
       "  0.0003445701823664639,\n",
       "  0.00034448716979626514,\n",
       "  0.00034818545485150936,\n",
       "  0.00035111038811835627,\n",
       "  0.0003458143265591581,\n",
       "  0.00034154288739256234,\n",
       "  0.00034647515170668173,\n",
       "  0.0003606842909505599,\n",
       "  0.00035974348258394957,\n",
       "  0.00036900219700400534,\n",
       "  0.0003687485401388892,\n",
       "  0.00037238710358060027,\n",
       "  0.0003771938570246208,\n",
       "  0.000375849988178378,\n",
       "  0.0003736552885980001,\n",
       "  0.00036886639545994753,\n",
       "  0.00037146774364189757,\n",
       "  0.00036834527782135166,\n",
       "  0.0003311235513800068,\n",
       "  0.0003319149504735093,\n",
       "  0.00033160749611931754,\n",
       "  0.00033369919974892044,\n",
       "  0.0003394683602114802,\n",
       "  0.00033865719537573287,\n",
       "  0.0003355699218047819,\n",
       "  0.00032214098555246565,\n",
       "  0.0003232250254185549,\n",
       "  0.00032800314389696026,\n",
       "  0.00032619068147532975,\n",
       "  0.0003296945349084257,\n",
       "  0.00033300221677443536,\n",
       "  0.0003383852746957881,\n",
       "  0.0003388924797376004,\n",
       "  0.0003404345486565007,\n",
       "  0.00034075337824106895,\n",
       "  0.000345986221478268,\n",
       "  0.00032903819435145727,\n",
       "  0.00033177476602432986,\n",
       "  0.00033395453307916883,\n",
       "  0.00034062702515514785,\n",
       "  0.00034327548167729097,\n",
       "  0.00034772840657859447,\n",
       "  0.00034574761288481726,\n",
       "  0.0003514197387542482,\n",
       "  0.0003405552676539358,\n",
       "  0.00034530201677409163,\n",
       "  0.0003456825883967475,\n",
       "  0.00034689087623643615,\n",
       "  0.0003479019752900842,\n",
       "  0.00034503238654350825,\n",
       "  0.0003655720790424955,\n",
       "  0.00036462346588702363,\n",
       "  0.00036686035987171456,\n",
       "  0.0003665541436052466,\n",
       "  0.00035388300657362466,\n",
       "  0.0003495816611555842,\n",
       "  0.00034992364892180754,\n",
       "  0.00035132907348949297,\n",
       "  0.0003547261039777841,\n",
       "  0.0003509942172917311,\n",
       "  0.0003485270052162907,\n",
       "  0.0003475698160318347,\n",
       "  0.00034713851138906514,\n",
       "  0.0003450616064305599,\n",
       "  0.0003410626156694854,\n",
       "  0.00033593312910181204,\n",
       "  0.0003409144451295292,\n",
       "  0.0003234684292340938,\n",
       "  0.00032087929132450655,\n",
       "  0.0003416079557349347,\n",
       "  0.00034089810536770167,\n",
       "  0.0003448695492822192,\n",
       "  0.0003429462860135401,\n",
       "  0.0003439268320145118,\n",
       "  0.0003468717115448271,\n",
       "  0.0003489608636084306,\n",
       "  0.0003486628512656438,\n",
       "  0.0003473351585439049,\n",
       "  0.00034644859563616394,\n",
       "  0.00033825050853858246,\n",
       "  0.00033692037215433897,\n",
       "  0.000332600354777557,\n",
       "  0.00033136615103313363,\n",
       "  0.0003364789558798455,\n",
       "  0.00034121195573955695,\n",
       "  0.0003374600883217705,\n",
       "  0.0003391657152901026,\n",
       "  0.00034238140776227924,\n",
       "  0.0003453458186492608,\n",
       "  0.0003418878754753914,\n",
       "  0.0003399564796072046,\n",
       "  0.00033743398662547955,\n",
       "  0.00033846294323434863,\n",
       "  0.00033780574023370094,\n",
       "  0.0003396724233127647,\n",
       "  0.00033715442034522796,\n",
       "  0.0003426339504216101,\n",
       "  0.0003404791258376796,\n",
       "  0.00033833600554830173,\n",
       "  0.0003400002228552355,\n",
       "  0.0003514932920755592,\n",
       "  0.00034676052053347087,\n",
       "  0.0003420743058555509,\n",
       "  0.0003472517783529838,\n",
       "  0.00034463551141170415,\n",
       "  0.00034360767178075066,\n",
       "  0.0003419758884230617,\n",
       "  0.0003366194892850362,\n",
       "  0.0003390405917774122,\n",
       "  0.00035271823126723244,\n",
       "  0.0003461443273894977,\n",
       "  0.00034292379895096953,\n",
       "  0.00033708979339224095,\n",
       "  0.00033056665390391595,\n",
       "  0.0003399649775023594,\n",
       "  0.0003431636939763116,\n",
       "  0.00034572068230731735,\n",
       "  0.0003485704996497309,\n",
       "  0.0003459497136788735,\n",
       "  0.0003431018095168186,\n",
       "  0.0003448723518683907,\n",
       "  0.0003429758420418261,\n",
       "  0.0003412745584410715,\n",
       "  0.00033846690007945725,\n",
       "  0.0003385575550292442,\n",
       "  0.00033384061901633,\n",
       "  0.0003356057108521613,\n",
       "  0.0003351370734560141,\n",
       "  0.00033713340547084124,\n",
       "  0.0003373559427819072,\n",
       "  0.00034017142034504995,\n",
       "  0.00034140943307328414,\n",
       "  0.0003465407574434254,\n",
       "  0.00034374699773326696,\n",
       "  0.00034041102045925705,\n",
       "  0.0003409183734648313,\n",
       "  0.00035021003830797105,\n",
       "  0.0003438470017996388,\n",
       "  0.0003500563022178328,\n",
       "  0.0003514671891225078,\n",
       "  0.0003533639551064367,\n",
       "  0.000349900167173602,\n",
       "  0.00034853109424092834,\n",
       "  0.0003467366902112982,\n",
       "  0.00034563492638563985,\n",
       "  0.0003414520723222297,\n",
       "  0.0003397311587604069,\n",
       "  0.0003407521125052996,\n",
       "  0.0003352266740473968,\n",
       "  0.0003310012865536591,\n",
       "  0.0003321021802351297,\n",
       "  0.0003330056282977969,\n",
       "  0.0003313216343239373,\n",
       "  0.0003324151404129339,\n",
       "  0.00032437673125841946,\n",
       "  0.0003219096897006067,\n",
       "  0.0003194394656745984,\n",
       "  0.00031806711268899567,\n",
       "  0.00031065693921142204,\n",
       "  0.0003162439745365748,\n",
       "  0.00031818829868368186,\n",
       "  0.0003089161887554742,\n",
       "  0.00030510355044949684,\n",
       "  0.00030646095078905533,\n",
       "  0.00030416427883778966,\n",
       "  0.0003001059913210399,\n",
       "  0.0003025582695969204,\n",
       "  0.0003021406508485803,\n",
       "  0.00030447417994774827,\n",
       "  0.00030673954749400347,\n",
       "  0.0002983040066591862,\n",
       "  0.0003016019481040087,\n",
       "  0.0002997133573673949,\n",
       "  0.0002965840515734758,\n",
       "  0.00029011339923007705,\n",
       "  0.0002921114309821913,\n",
       "  0.0002895592785167325,\n",
       "  0.00029205069268164427,\n",
       "  0.0002918197830928202,\n",
       "  0.00030528744842753437,\n",
       "  0.0003053576955094193,\n",
       "  0.00030642541143968515,\n",
       "  0.000307515351238808,\n",
       "  0.0003076111418555526,\n",
       "  0.0003126053561967702,\n",
       "  0.00031240988205877626,\n",
       "  0.00028526177287507934,\n",
       "  0.0002913290812808205,\n",
       "  0.00029455585182051875,\n",
       "  0.00028577104339886294,\n",
       "  0.0002804242256762148,\n",
       "  0.00028553777017628737,\n",
       "  0.00028450954493555963,\n",
       "  0.00028544501141310245,\n",
       "  0.0002833094213886733,\n",
       "  0.0002766714624673823,\n",
       "  0.0002761973707585374,\n",
       "  0.0002774210015921996,\n",
       "  0.00028074295904832963,\n",
       "  0.00027824296556388297,\n",
       "  0.00026771124603614373,\n",
       "  0.0002765644391247294,\n",
       "  0.000282123226193628,\n",
       "  0.00029486576420539173,\n",
       "  0.0002952962076121413,\n",
       "  0.00029421414231157084,\n",
       "  0.0002942881198966673,\n",
       "  0.0002944508165775497,\n",
       "  0.0002951091577610047,\n",
       "  0.0003051767831140631,\n",
       "  0.00030689952958277836,\n",
       "  0.0003098515569092105,\n",
       "  0.00030834040031896926,\n",
       "  0.0003053853447698691,\n",
       "  0.0003083785925018403,\n",
       "  0.0003086681855903428,\n",
       "  0.00031072504356068856,\n",
       "  0.000291626976027148,\n",
       "  0.0002928176033776385,\n",
       "  0.0002953032083473366,\n",
       "  0.00029519699586115825,\n",
       "  0.0003012440221965426,\n",
       "  0.0002754000820990859,\n",
       "  0.00026718953389312045,\n",
       "  0.000270833263311977,\n",
       "  0.0002775042436199746,\n",
       "  0.00027652066200607435,\n",
       "  0.00027643478009538075,\n",
       "  0.0002776635871408693,\n",
       "  0.000274234857402379,\n",
       "  0.00027555400697081445,\n",
       "  0.00027929927934524707,\n",
       "  0.00027949245703202605,\n",
       "  0.000282178838978299,\n",
       "  0.0002811738990368415,\n",
       "  0.00025711837703465895,\n",
       "  0.0002594903725640263,\n",
       "  0.0002621766175535625,\n",
       "  0.0002514394796227417,\n",
       "  0.000260896378647329,\n",
       "  0.0002596312960394203,\n",
       "  0.00026004654472688,\n",
       "  0.00025270499944372567,\n",
       "  0.00026337769529468047,\n",
       "  0.00026360312950378395,\n",
       "  0.000272047142126085,\n",
       "  0.0002710413282924864,\n",
       "  0.00027017458412839597,\n",
       "  0.00027183863959514876,\n",
       "  0.00027111391636571477,\n",
       "  0.0002817631065562483,\n",
       "  0.0002823034724357296,\n",
       "  0.00028198870839566206,\n",
       "  0.0002773059156223419,\n",
       "  0.00027600155333208545,\n",
       "  0.0002739059457623746,\n",
       "  0.0002756893608498116,\n",
       "  0.0002767068843961885,\n",
       "  0.0002759316369622822,\n",
       "  0.0002937103945377381,\n",
       "  0.00029156636100730954,\n",
       "  0.00029673339974149916,\n",
       "  0.000297232420184423,\n",
       "  0.00030029269408685306,\n",
       "  0.0002992050422224904,\n",
       "  0.0002965429237808221,\n",
       "  0.00029401216659487094,\n",
       "  0.0002948453937898279,\n",
       "  0.0002916049448218432,\n",
       "  0.0002911212194320439,\n",
       "  0.00029151810247241087,\n",
       "  0.0002931564464675206,\n",
       "  0.00029329145444990533,\n",
       "  0.00029367637121089344,\n",
       "  0.000293948683905099,\n",
       "  0.0002991830100350995,\n",
       "  0.0003088678172056761,\n",
       "  0.00030599247042951516,\n",
       "  0.000311139514151033,\n",
       "  0.0003092902822727994,\n",
       "  0.0003071647645117282,\n",
       "  0.00029744555798098273,\n",
       "  0.00029433070668612854,\n",
       "  0.0002933631614778642,\n",
       "  0.0002854135708740909,\n",
       "  0.00029121714440984186,\n",
       "  0.0002930230712635373,\n",
       "  ...]}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print('best_score:%f '% cv_result['auc-mean'][-10:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best estimator's num: 9431\n"
     ]
    }
   ],
   "source": [
    "best_n_estimators = len(cv_result['auc-mean'])\n",
    "print(\"best estimator's num: %d\" % n_estimators)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. num_leaves & max_depth=7\n",
    "num_leaves建议70-80，搜索区间50-80,值越大模型越复杂，越容易过拟合\n",
    "相应的扩大max_depth=7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num_leaves_s = range(50,90,10) #50,60,70,80\n",
    "# tuned_parameters = dict(num_leaves = num_leaves_s)\n",
    "# print(tuned_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned_parameters {'num_leaves': range(50, 90, 10)}\n",
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done   1 tasks      | elapsed: 158.4min\n",
      "[Parallel(n_jobs=6)]: Done   3 out of  12 | elapsed: 200.6min remaining: 601.8min\n",
      "[Parallel(n_jobs=6)]: Done   5 out of  12 | elapsed: 220.3min remaining: 308.4min\n",
      "[Parallel(n_jobs=6)]: Done   7 out of  12 | elapsed: 301.5min remaining: 215.4min\n",
      "[Parallel(n_jobs=6)]: Done   9 out of  12 | elapsed: 403.6min remaining: 134.5min\n",
      "[Parallel(n_jobs=6)]: Done  12 out of  12 | elapsed: 417.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11.2 s, sys: 2.77 s, total: 14 s\n",
      "Wall time: 6h 57min 31s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      class_weight=None, colsample_bytree=0.7,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=32,\n",
       "                                      max_depth=7, min_child_samples=20,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=9431,\n",
       "                                      n_jobs=6, num_leaves=31,\n",
       "                                      objective='regression', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.7,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='warn', n_jobs=6, param_grid={'num_leaves': range(50, 90, 10)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=10)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7,\n",
    "          'max_bin': 32, #原始特征为整数，很少超过100\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "          \n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "num_leaves_s = range(50,90,10) #50,60,70,80\n",
    "tuned_parameters = dict( num_leaves = num_leaves_s)\n",
    "print('tuned_parameters',tuned_parameters)\n",
    "grid_search = GridSearchCV(lg, n_jobs=6, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=10, refit = False)\n",
    "grid_search.fit(train_X , train_Y)\n",
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(model_path + 'grid_search.pkl','wb') as fw:\n",
    "        pk.dump(grid_search,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump grid_search error,',e)\n",
    "try:\n",
    "    # 序列化到磁盘中\n",
    "    with open(model_path + 'lg.pkl','wb') as fw:\n",
    "        pk.dump(lg,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg error,',e)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "减小 max_bin ，提高训练速度。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.618939772166656\n",
      "{'num_leaves': 50}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'num_leaves': 50}"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([6121.50513005, 7995.63257233, 6844.20159356, 7261.39334019]),\n",
       " 'std_fit_time': array([154.23484005, 988.78289402, 389.01148033, 429.18436915]),\n",
       " 'mean_score_time': array([5902.43924061, 4038.8156933 , 3843.95129553, 4440.51144163]),\n",
       " 'std_score_time': array([ 273.00451681,  826.16543892, 1307.57022528,  682.39449984]),\n",
       " 'param_num_leaves': masked_array(data=[50, 60, 70, 80],\n",
       "              mask=[False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'num_leaves': 50},\n",
       "  {'num_leaves': 60},\n",
       "  {'num_leaves': 70},\n",
       "  {'num_leaves': 80}],\n",
       " 'split0_test_score': array([-0.6188905 , -0.61991655, -0.62068165, -0.622052  ]),\n",
       " 'split1_test_score': array([-0.61904019, -0.6198853 , -0.62127817, -0.62263506]),\n",
       " 'split2_test_score': array([-0.61888863, -0.61953435, -0.62098739, -0.62211978]),\n",
       " 'mean_test_score': array([-0.61893977, -0.61977873, -0.6209824 , -0.62226895]),\n",
       " 'std_test_score': array([7.10106716e-05, 1.73274611e-04, 2.43551473e-04, 2.60357401e-04]),\n",
       " 'rank_test_score': array([1, 2, 3, 4], dtype=int32)}"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEHCAYAAABm9dtzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3gVZdrH8e9NIPRepAYQAopIM1LEAqwFK7oiIjZs2NBFd111dXdZX33X3XVXRVEXe0MQC2JZsYBKVQKEFoWE0AJCCBB6SbnfP86wb4wBIpzDyUl+n+vKdTIzz5m5H4bkl3lmzoy5OyIiIuFQIdoFiIhI2aFQERGRsFGoiIhI2ChUREQkbBQqIiISNhWjXUA0NWjQwFu1ahXtMkREYsrcuXOz3b1hccvKdai0atWK5OTkaJchIhJTzGzVgZZp+EtERMJGoSIiImGjUBERkbBRqIiISNgoVEREJGwUKiIiEjYKFRERCRuFiohIOeLujPtuNSuyd0Zk/QoVEZFyYufePEaMT+G+9xbx6syVEdlGuf5EvYhIefHD+m3c9uY8Vmbv5LdnteP2vm0jsh2FiohIGebuTEjO5I8fLKZW1Uq8cWMPTmnTIGLbU6iIiJRRu/bl8eDExbw3by2929bnicu70rBm5YhuU6EiIlIGLduwndvenMfyjTsYcWYid/RLJK6CRXy7ChURkTLmnbmZ/HHiYqpXjuONG3rQu23khruKUqiIiJQRu/fl86cPFjNhbiY9j63HqMFdaVSrylGtQaEiIlIGpGeFhrvSsnZwZ7+2/ObMdkdluKsohYqISIx7f34mD7y/mKqV4nj1uu6c3q7YhzIeFQoVEZEYtSc3n5GTljBuzhq6t67HU1d05ZijPNxVlEJFRCQGLd+4g9vfnMcP67dze9823HVmOyrGRf8mKQoVEZEY80HKWv7w3iLiK1bgletOpk/7RtEu6b8UKiIiMWJPbj4PfZTK2G9Xk9SyLk8N6UqT2lWjXdZPRPRYycz6m9lSM0s3s/sO0GaQmaWa2RIzGxvM62Jms4J5C83s8kLt3wzWudjMXjKzSsF8M7NRwbYWmlm3SPZNRORoWpG9k0uemcnYb1dzyxlteGtYz1IXKBDBIxUziwNGA2cBmcAcM5vk7qmF2iQC9wO93X2Lme0/htsFXOPuaWbWFJhrZpPdPQd4E7gqaDcWuBF4FjgXSAy+egTzekSqfyIiR8tHC9dx37uLqBhnvDQ0iX7HHRPtkg4oksNf3YF0d88AMLNxwAAgtVCbm4DR7r4FwN2zgtdl+xu4+zozywIaAjnu/sn+ZWb2HdA8mBwAvObuDsw2szpm1sTdf4xYD0VEImhPbj6PfPw9r89eRbeEOjw1pBvN6pS+o5PCIhkqzYA1haYz+fmRQzsAM5sBxAEj3f3Twg3MrDsQDywvMr8ScDXwm4NsrxnwY5H3DQOGASQkJPzSPomIHBWrNu3k9rHzWLx2G8NOP5Z7zmlPpVJwddehRDJUivsopxez/USgD6Ejjmlm1jEY5sLMmgCvA9e6e0GR9z4DfOPu037B9nD3McAYgKSkpJ8tFxGJtv8s+pHfv7OQChWMF65J4swOpXe4q6hIhkom0KLQdHNgXTFtZrt7LrDCzJYSCpk5ZlYL+Bh40N1nF36Tmf2Z0HDYzb9weyIipdbevHz++skPvDJzJZ1b1GH0kK40r1st2mX9IpE8lpoDJJpZazOLBwYDk4q0mQj0BTCzBoSGwzKC9u8TOkcyofAbzOxG4BzgiiJHL5OAa4KrwHoCW3U+RURixZrNu7jsuVm8MnMlN5zamgk394q5QIEIHqm4e56ZDQcmEzpf8pK7LzGzh4Bkd58ULDvbzFKBfOAed99kZlcBpwP1zWxosMqh7p4CPAesAmaZGcB77v4Q8AlwHpBO6Oqx6yLVNxGRcPp08XrueWcBAP+++iTOOaFxlCs6fBa6WKp8SkpK8uTk5GiXISLl1L68Ah79zw+8NGMFnZrXZvSQbrSoV/qPTsxsrrsnFbdMn6gXEYmCNZt3Mfyt+SxYk8PQU1px/3nHUbliXLTLOmIKFRGRo+zz1A389u0U3OHZK7tx7olNol1S2ChURESOktz8Av7+6Q88P20FHZvVYvSQbrSsXz3aZYWVQkVE5ChYm7Ob4WPnMX91Dtf0askD5x9fJoa7ilKoiIhE2JQfNnD32wvIy3eeHtKVCzo1jXZJEaNQERGJkNz8Ah77bCn//jqDDk1qMfrKbrRuULaGu4pSqIiIRMCPW3dzx9j5JK/awpU9EvjjBR2oUqnsDXcVpVAREQmzqUuzuHt8CvvyCnhycBcGdGkW7ZKOGoWKiEiY5OUX8K/Pl/HMV8s5rnFNRl/ZjTYNa0S7rKNKoSIiEgbrt+7hzrfm893KzVzRvQV/vvCEcjHcVZRCRUTkCH2zbCN3jU9hd24+T1zehYu7lp/hrqIUKiIihym/wHnii2U8PTWddo1Cw11tG5Wv4a6iFCoiIocha9se7hw3n9kZmxmU1Jy/XNSRqvHlb7irKIWKiMgvNCM9m9+Mm8/Ovfn887LOXHpS82iXVGooVERESii/wBn1ZRqjpqTRtmEN3rqpG4nH1Ix2WaWKQkVEpASytu9hxLgUZi7fxK+7NePhiztSLV6/QovSv4iIyCHMXJ7Nb8alsH1PLn8f2IlBSS2iXVKppVARETmA/AJn9NR0nvhiGa0bVOeNG3rQvrGGuw5GoSIiUozsHXu5a3wK09KyubhLUx655ESqV9avzEPRv5CISBGzMzZx51vz2bo7l0d/fSKXn9wCM4t2WTFBoSIiEigocJ79ejn//GwprepX59Xru3N8k1rRLiumVIjkys2sv5ktNbN0M7vvAG0GmVmqmS0xs7HBvC5mNiuYt9DMLi/UfniwPjezBoXm9zGzrWaWEnz9KZJ9E5GyZdOOvQx9ZQ7/mLyU8zs1ZdIdpypQDkPEjlTMLA4YDZwFZAJzzGySu6cWapMI3A/0dvctZtYoWLQLuMbd08ysKTDXzCa7ew4wA/gI+KqYzU5z9wsi1ScRKZvmrNzMHWPns3nXPh65pCNDuidouOswRXL4qzuQ7u4ZAGY2DhgApBZqcxMw2t23ALh7VvC6bH8Dd19nZllAQyDH3ecH64tg6SJSHhQUOP/+JoPHPltKi7pVee/WU+jYrHa0y4ppkQyVZsCaQtOZQI8ibdoBmNkMIA4Y6e6fFm5gZt2BeGB5CbbZy8wWAOuA37n7kqINzGwYMAwgISGhZD0RkTJny8593P12ClOXbuT8E5vw6KUnUrNKpWiXFfMiGSrFHUp4MdtPBPoAzYFpZtYxGObCzJoArwPXunvBIbY3D2jp7jvM7DxgYrDunxbgPgYYA5CUlFS0HhEpB+au2szwsfPZtGMf/zPgBK7q2VKjH2ESyRP1mUDhj502J3QEUbTNB+6e6+4rgKUEQWBmtYCPgQfdffahNubu29x9R/D9J0ClwifyRUTcnTHfLOfyf8+mUlwF3r31FK7u1UqBEkaRDJU5QKKZtTazeGAwMKlIm4lAX4AgANoBGUH794HX3H1CSTZmZo0t+J8RDJlVADaFpSciEvNydu3jpteS+d9PfuCsDsfw0Z2ncmJznT8Jt4gNf7l7npkNByYTOl/ykrsvMbOHgGR3nxQsO9vMUoF84B5332RmVwGnA/XNbGiwyqHunmJmdwK/BxoDC83sE3e/ERgI3GpmecBuYLC7a3hLRJi3egt3jJ1P1vY9jLywA9eeoqOTSLHy/Hs3KSnJk5OTo12GiESIu/Pi9BU8+p8faFy7CqOHdKNzizrRLivmmdlcd08qbpk+US8iZdLWXbn87p0FfJ66gbM7HMM/BnamdjVd3RVpChURKXMWrMnh9rHzWL91D3+8oAPX99Zw19GiUBGRMsPdeWXmSv73k+9pVLMKE27pRdeEutEuq1xRqIhImbB1dy73vrOQT5es58zjG/HYZZ2pUy0+2mWVOwoVEYl5izK3cvvYeazL2c0D5x3Pjae11nBXlChURCRmuTuvz17Fwx99T4Ma8Yy/uRcntdRwVzQpVEQkJm3bk8v97y7i40U/0rd9Q/41qAt1q2u4K9oUKiIScxavDQ13ZW7ZzX3nHsew046lQgUNd5UGChURiRnuzpvfruahj1KpVy2eccN6cnKretEuSwpRqIhITNixN4/731vEhwvWcUa7hvxrUGfq16gc7bKkCIWKiJR6qeu2cfvYeazatJN7zmnPrWe00XBXKaVQEZFSy90ZN2cNIyctoXbVSrx1U096HFs/2mXJQShURKRU2rk3jwfeX8TElHWcltiAxy/vQgMNd5V6ChURKXWWrt/ObW/OZUX2Tn57Vjtu79tWw10xQqEiIqXK28lr+NMHi6lZpRJv3NiDU9roAa6xRKEiIqXCrn15/HHiEt6dl0nvtvV54vKuNKyp4a5Yo1ARkahL27Cd296cR/rGHYw4M5E7+iUSp+GumKRQEZGoenduJg9OXEz1ynG8cUMPerfVcFcsU6iISFTs3pfPnyct5u3kTHoeW49Rg7vSqFaVaJclR0ihIiJHXXrWDm5/cx7LsrZzZ7+2/ObMdhruKiMqRHLlZtbfzJaaWbqZ3XeANoPMLNXMlpjZ2GBeFzObFcxbaGaXF2o/PFifm1mDQvPNzEYFyxaaWbdI9k1EDs/E+Wu56OnpZO/Yy6vXdefus9srUMqQiB2pmFkcMBo4C8gE5pjZJHdPLdQmEbgf6O3uW8ysUbBoF3CNu6eZWVNgrplNdvccYAbwEfBVkU2eCyQGXz2AZ4NXESkF9uTm85cPl/DWd2vo3roeT13RlWM03FXmRHL4qzuQ7u4ZAGY2DhgApBZqcxMw2t23ALh7VvC6bH8Dd19nZllAQyDH3ecH6yu6vQHAa+7uwGwzq2NmTdz9x4j0TkRKLGPjDm57cx4/rN/O7X3bcNeZ7agYF9GBEomSSIZKM2BNoelMfn7k0A7AzGYAccBId/+0cAMz6w7EA8sPY3vNAIWKSBRNWrCO+99dSHzFCrxy3cn0ad/o0G+SmBXJUClukNSL2X4i0AdoDkwzs47BMBdm1gR4HbjW3QvCsD3MbBgwDCAhIeEQqxSRw7Vzbx4PfZjK+OQ1JLWsy1NDutKkdtVolyURFslQyQRaFJpuDqwrps1sd88FVpjZUkIhM8fMagEfAw+6++wwbQ93HwOMAUhKSvpZ6IjIkZu3egt3jU9h9eZd3N63DSPObEclDXeVC5Hcy3OARDNrbWbxwGBgUpE2E4G+AMGVXO2AjKD9+4TOkUwo4fYmAdcEV4H1BLbqfIrI0ZWXX8ATXyzjsudmkZfvjB/Wi3vOOU6BUo4cck+bWRszqxx838fM7jSzOod6n7vnAcOBycD3wNvuvsTMHjKzi4Jmk4FNZpYKTAXucfdNwCDgdGComaUEX12CGu40s0xCRyILzeyFYF2fABlAOvA8cFtJ/xFE5MitzN7JwOdm8cQXaQzo3JT/jDiN7q31qN/yxkIXSx2kgVkKkAS0IhQCk4D27n5exKuLsKSkJE9OTo52GSIxzd2ZkJzJyA+XULGC8cglJ3Jh56bRLksiyMzmuntScctKck6lwN3zzOwS4Al3f8rM5oe3RBGJRZt37uP+9xYyeckGeh1bn38O6kzTOjoZX56VJFRyzewK4FrgwmBepciVJCKx4JtlG/ndhAXk7MrlgfOO54ZTW+tBWlKiULkOuAV4xN1XmFlr4I3IliUipdWe3Hwe/c8PvDJzJe2OqcEr13WnQ9Na0S5LSolDhkpwW5U7AcysLlDT3R+NdGEiUvqkrtvGiPHzWbZhB9f1bsW9/Y+jSqW4aJclpcghQ8XMvgIuCtqmABvN7Gt3vzvCtYlIKVFQ4LwwPYPHJi+jdrVKvHp9d85o1zDaZUkpVJLhr9ruvs3MbgRedvc/m9nCSBcmIqXDupzd/PbtBczK2MQ5JxzDX3/diXrV46NdlpRSJQmVisHtUgYBD0S4HhEpRT5csI4H3l9EXoHz90s7cVlS8+Ju5iryXyUJlYcIfT5lhrvPMbNjgbTIliUi0bRtTy4jP1jCe/PX0jWhDk9c3oWW9atHuyyJASU5UT8BmFBoOgO4NJJFiUj0fLdiM3eNT2H9tj2MODOR4X3b6jb1UmIlOVHfHHgK6E3orr/Tgd+4e2aEaxORo2hfXgFPfrmMZ79aTot61ZhwSy+6JdSNdlkSY0oy/PUyMBa4LJi+Kph3VqSKEpGjKz1rB3eNT2HR2q1cntSCP13YgeqVI3kTcymrSvK/pqG7v1xo+hUzGxGpgkTk6HF33vx2NQ9/nErVSnE8d9VJ9O/YONplSQwrSahkm9lVwFvB9BXApsiVJCJHw8bte7n33YVM+SGL09s15LGBnWikZ8bLESpJqFwPPA08TuicykxCt24RkRj15fcb+P07C9m+N4+RF3bgml6tdN8uCYuSXP21mtAn6v8rGP56IlJFiUhk7NqXxyMff8+b367m+Ca1eGtwF9odUzPaZUkZcrhn4u5GoSISUxZm5jBiXAorNu3k5tOP5e6z21G5ou7bJeF1uKGi42SRGJFf4Dz39XIe/3wZDWtW5s0be3BKmwbRLkvKqMMNlYM/LlJESoU1m3dx99spzFm5hQs6NeGRi0+kdjU9Dkki54ChYmbbKT48DNCj3URKMXfn/flr+dMHSzDgicu7MKBLU923SyLugKHi7jp7JxKDcnbt44GJi/l44Y90b1WPfw7qTIt61aJdlpQT+sisSBkyMz2bu99eQPaOvfy+f3tuPr0NcbpUWI6iiN4lzsz6m9lSM0s3s/sO0GaQmaWa2RIzGxvM62Jms4J5C83s8kLtW5vZt2aWZmbjzSw+mD/UzDaaWUrwdWMk+yZSmuzNy+eRj1MZ8sK3VKscx/u39ea2Pm0VKHLURexIxczigNGE7hGWCcwxs0nB44n3t0kE7gd6u/sWM2sULNoFXOPuaWbWFJhrZpPdPQf4G/C4u48zs+eAG4Bng/eNd/fhkeqTSGm0dP12fjNuPj+s387VPVvyh/OOp2q8LhWW6IjkkUp3IN3dM9x9HzAOGFCkzU3AaHffAuDuWcHrMndPC75fB2QBDS10lrEf8E7w/leBiyPYB5FSq6DAeWn6Ci58ejrZO/by0tAk/ufijgoUiaqS3Pq+uKvAtgLJwG+D56sUpxmwptB0JtCjSJt2wTZmAHHASHf/tMj2uwPxwHKgPpDj7nmF1tmsUPNLzex0YBlwl7sX3v7+9Q0DhgEkJCQcoHSR0m3Dtj38bsICpqVlc+bxjXj00k40qFE52mWJlGj461/AOkK3vzdgMNAYWAq8BPQ5wPuKG8wtGk4VgcRgHc2BaWbWMRjmIniM8evAte5eYMVfD7l/nR8Cb7n7XjO7hdBRTL+fNXYfA4wBSEpK0udtJOZ8uvhH7ntvEXty83nkko4M6Z6gS4Wl1ChJqPR398JHGGPMbLa7P2RmfzjI+zKBFoWmmxMKp6JtZrt7LrDCzJYSCpk5ZlYL+Bh40N1nB+2zgTpmVjE4WvnvOt298J2Tnyd07kWkzNixN4+HPlzC28mZdGpem8cv70KbhjWiXZbIT5TknEpBcIVWheBrUKFlB/tLfw6QGFytFU/oCGdSkTYTgb4AZtaA0HBYRtD+feC14HHGoY25OzAVGBjMuhb4IHh/k0LrvQj4vgR9E4kJc1dt4bwnp/HO3EyG923Lu7eeokCRUqkkRypXAk8CzwTTs4CrzKwqcMArrdw9z8yGA5MJnS95yd2XmNlDQLK7TwqWnW1mqUA+cI+7bwqe33I6UN/MhgarHOruKcC9wDgzexiYD7wYLL/TzC4C8oDNwP73icSsvPwCnpqSztNT02lSuwrjb+7Fya3qRbsskQOy0B//5VNSUpInJydHuwyRYq3M3smI8SmkrMnh192aMfKiE6hVRfftkugzs7nunlTcspJc/dUceAroTWi4azrwG3fPDGuVIgKE7ts1fs4aHvoolUpxFXh6SFcu6NQ02mWJlEhJhr9eJnTl12XB9FXBvLMiVZRIebV55z7ue3chn6Vu4JQ29fnnoM40qa37t0rsKEmoNHT3lwtNvxI8+VFEwuirpVnc885Ctu7K5cHzj+f63q31iF+JOSUJlezgxPlbwfQVwKaDtBeRX2BPbj6P/ucHXpm5knbH1ODV67rToWmtaJclclhKEirXA08DjxM6pzITuC6SRYmUF0vWbWXEuBTSsnZwfe/W/L5/e6pU0m1WJHYdMlTcfTWhz338VzD8pWfUixymggLn+WkZPPbZUupWi+f1G7pzWmLDaJclcsQO9y7Fd6NQETks63J2c/fbKczO2Ez/Exrz11+fSN3q8dEuSyQsDjdUdPZQ5DBMWrCOB95fREGB8/eBnbjspOa6b5eUKYcbKuX3E5Mih2Hbnlz+NHExE1PW0S2hDo9f3oWW9atHuyyRsDtgqBzglvcQOkrRhfMiJfRtxibufnsB67ft4e6z2nFbnzZUjIvoQ1dFouaAoeLuNY9mISJlzb68Ah7/YhnPfb2clvWq8c4tveiaUDfaZYlEVMQeJyxSnqVn7WDE+PksXruNK7q34MHzO1C9sn7cpOzT/3KRMHJ33pi9ikc++Z6qleL499Uncc4JjaNdlshRo1ARCZON2/fy+3cWMHXpRs5o15B/DOxEo1pVol2WyFGlUBEJgy9SN3DvuwvZsTePv1x0Atf0aqlLhaVcUqiIHIFd+/J4+OPvGfvtajo0qcW4wV1IPEbXuEj5pVAROUwL1uQwYnwKKzft5OYzjuXus9pRuaLu2yXlm0JF5BfKL3Ce/SqdJ75Io1HNyoy9sSe92tSPdlkipYJCReQXWLN5F3eNTyF51RYu7NyUhwd0pHY1PeJXZD+FikgJuDvvzVvLnyctwYAnB3dhQJdm0S5LpNRRqIgcQs6ufTzw/mI+XvQj3VvX41+DOtO8brVolyVSKkX0BkRm1t/MlppZupndd4A2g8ws1cyWmNnYYF4XM5sVzFtoZpcXat/azL41szQzG29m8cH8ysF0erC8VST7JuXDjPRs+j8xjc9S13Nv/+N466aeChSRg4hYqJhZHDAaOBfoAFxhZh2KtEkE7gd6u/sJwIhg0S7gmmBef+AJM6sTLPsb8Li7JwJbgBuC+TcAW9y9LaGnVP4tUn2Tsm9Pbj4Pf5TKlS98S/XKcbx/W29u7dOGOD0zXuSgInmk0h1Id/cMd98HjAMGFGlzEzDa3bcAuHtW8LrM3dOC79cBWUBDC32arB/wTvD+V4GLg+8HBNMEy39l+vSZHIal67dz8egZvDB9BVf3bMlHd5xGx2a1o12WSEyI5DmVZsCaQtOZQI8ibdoBmNkMIA4Y6e6fFm5gZt2BeGA5UB/Icfe8Quvcf7b0v9tz9zwz2xq0zy6yvmHAMICEhIQj6J6UNQUFzsszV/K3T3+gVpWKvDz0ZPoe1yjaZYnElEiGSnFHCUWfz1IRSAT6AM2BaWbW0d1zAMysCfA6cK27FxzgyGP/OkuyPdx9DDAGICkpSQ8bEwA2bNvD7yYsYFpaNmce34hHL+1EgxqVo12WSMyJZKhkAi0KTTcH1hXTZra75wIrzGwpoZCZY2a1gI+BB919dtA+G6hjZhWDo5XC69y/vUwzqwjUBjZHoF9Sxvxn0Y/c//4i9uYW8L+XnMgV3Vvovl0ihymS51TmAInB1VrxwGBgUpE2E4G+AGbWgNBwWEbQ/n3gNXefsL+xuzswFRgYzLoW+CD4flIwTbB8StBepFg79ubxuwkLuPXNebSsV42P7zyVIT0SFCgiRyBiRyrBeY3hwGRC50tecvclZvYQkOzuk4JlZ5tZKpAP3OPum8zsKuB0oL6ZDQ1WOdTdU4B7gXFm9jAwH3gxWP4i8LqZpRM6Qhkcqb5J7Ju7ajN3jV9A5pZd3NGvLXf+KpFKesSvyBGz8vzHfFJSkicnJ0e7DDmKcvMLeGpKOk9PSaNZ3ao8PqgLSa3qRbsskZhiZnPdPam4ZfpEvZQbK7J3MmJ8CgvW5HBpt+aMvKgDNavovl0i4aRQkTLP3Rk/Zw0PfZRKpbgKjB7SjfM7NYl2WSJlkkJFyrRNO/Zy33uL+Dx1A73b1uefl3WhcW094lckUhQqUmZNXZrFPRMWsm13Lg+efzzX925NBd1mRSSiFCpS5uzJzeevn3zPq7NW0f6Ymrx+Q3eOb1Ir2mWJlAsKFSlTFq/dyojxKaRn7eCGU1tzzzntqVJJj/gVOVoUKlIm7Nybx0vTVzBqShr1qsfzxg09ODWxQbTLEil3FCoS07bvyeW1Wat4YVoGW3blcv6JTXj44o7UrR4f7dJEyiWFisSkrbtyeXnmCl6avoJte/Lod1wj7ujXlq4JdaNdmki5plCRmLJl5z5emrGCV2asZPvePM7qcAx39kvkxOZ63olIaaBQkZiwacdenp+2gtdnrWTnvnzOO7Exw/sm0qGpruoSKU0UKlKqZW3fw/PfZPDG7NXsycvngk5NGd63Le0b14x2aSJSDIWKlErrt+7hua+X89Z3q8nNL+DiLs24rW9b2jaqEe3SROQgFCpSqqzN2c2zX6Xz9pxMCtz5dbdm3NanLa0aVI92aSJSAgoVKRXWbN7FM1+l887cTAAGntSC2/q0oUW9alGuTER+CYWKRNWK7J2MnprO+/PXEmfGFd0TuPmMNjSrUzXapYnIYVCoSFSkZ23n6SnpTFqwjkpxFbimV0tuPr2N7iAsEuMUKnJULV2/naempPHxoh+pUjGOG087lhtPa02jmgoTkbJAoSJHxZJ1W3nqy3Q+XbKe6vFx3HpGG244tTX1a1SOdmkiEkYKFYmohZk5jPoynS++30DNyhW5s19brj+1NXWq6d5cImVRREPFzPoDTwJxwAvu/mgxbQYBIwEHFrj7kGD+p0BPYLq7X1CofT/gMSAemAvc4O55ZtYH+ABYETR9z90filDX5BDmrtrCU1PS+GrpRmpXrcTdZ7Xj2lNaUbuqngkvUpZFLFTMLA4YDZwFZAJzzGySu6cWapMI3A/0dvctZtao0Cr+AVQDbi7UvgLwKvArd19mZg8B1wIvBk2mFQ4gOfq+W7GZUV+mMbR6PDcAAA7YSURBVD09m7rVKvH7/u25umdLalZRmIiUB5E8UukOpLt7BoCZjQMGAKmF2twEjHb3LQDunrV/gbt/GRx9FFYf2Ovuy4LpzwmF0otI1Lg7szI2MerLNGZnbKZBjXj+cN5xXNmjJdUra4RVpDyJ5E98M2BNoelMoEeRNu0AzGwGoSGyke7+6UHWmQ1UMrMkd08GBgItCi3vZWYLgHXA79x9yRH2QQ7C3ZmWls2oL9NIXrWFRjUr86cLOnBF9wSqxutpiyLlUSRDxYqZ58VsPxHoAzQHpplZR3fPKW6F7u5mNhh43MwqA58BecHieUBLd99hZucBE4N1/7Qos2HAMICEhIRf3CkJhcnUpVmM+jKdlDU5NKldhYcGnMCgpBZ6dK9IORfJUMnkp0cRzQkdQRRtM9vdc4EVZraUUBDMOdBK3X0WcBqAmZ1NcLTj7tsKtfnEzJ4xswbunl3k/WOAMQBJSUlFQ04Owt35PHUDo6aksXjtNprVqcr/XnIil57UjMoVFSYiEtlQmQMkmllrYC0wGBhSpM1E4ArgFTNrQCggMg62UjNr5O5ZwZHKvcAjwfzGwIbgaKY7UAHYFM4OlVcFBc6nS9Yz6ss0fli/nZb1q/H3gZ24pGszKsVViHZ5IlKKRCxUgst8hwOTCZ0vecndlwRXbCW7+6Rg2dlmlgrkA/e4+yYAM5sGHAfUMLNMQpcOTwbuMbMLCIXGs+4+JdjkQOBWM8sDdgOD3V1HIkcgv8D5aOE6np6STlrWDo5tWJ1/DerMRZ2bUlFhIiLFsPL8ezcpKcmTk5OjXUapk5dfwKQF63h6ajoZG3eS2KgGd/wqkfNPbEJcheJOlYlIeWJmc909qbhlut5T/is3v4D3561l9FfprNq0i+Ma1+SZK7vR/4TGVFCYiEgJKFSEvXn5vDt3Lc98lU7mlt10bFaLMVefxJnHH6MwEZFfRKFSju3Jzeft5DU8+9Vyfty6hy4t6vA/AzrSp31DzBQmIvLLKVTKod378hn73Wr+/fVysrbvJallXf52aSdOS2ygMBGRI6JQKUd27s3jzW9XMeabDLJ37KPnsfV4YnAXeh1bX2EiImGhUCkHtu/J5bVZq3hx+go279zHaYkNuKNfIt1b14t2aSJSxihUyrCtu3N5deZKXpy+gq27c+nTviF39EvkpJZ1o12aiJRRCpUyKGfXPl6avoKXZ6xk+948zjz+GO78VVs6Na8T7dJEpIxTqJQhm3bs5YXpK3ht5kp27sun/wmNGd6vLR2b1Y52aSJSTihUyoCN2/fy/LQMXp+1ij15+Zx/YhOG92vLcY1rRbs0ESlnFCoxbMO2PTz39XLGfrua3PwCBnRpxu1929C2Uc1olyYi5ZRCJQatzdnNc18tZ3zyGvILnEu6NuP2vm1p3aB6tEsTkXJOoRJD1mzexTNfLeeduaEHag48qTm3ntGWhPrVolyZiEiIQiUGrMzeyeip6bw3fy1xZgw+OYFb+rShWZ2q0S5NROQnFCql2PKNOxg9JZ2JKWupFFeBq3u25JYz2tC4dpVolyYiUiyFSim0bMN2npqSzkcL11GlYhw3nNqam04/lkY1FSYiUropVEqR1HXbeGpKGv9ZvJ7q8XHcfHobbjytNQ1qVI52aSIiJaJQKQUWZW5l1JQ0Pk/dQM3KFbmjX1uu792autXjo12aiMgvolCJonmrt/DUl2lMXbqRWlUqcteZ7RjauxW1q1aKdmkiIodFoRIFc1ZuZtSXaUxLy6ZutUrcc057runVkppVFCYiEtsUKkeJuzM7IxQmszI2Ub96PPefexxX9WxJ9craDSJSNkT0t5mZ9QeeBOKAF9z90WLaDAJGAg4scPchwfxPgZ7AdHe/oFD7fsBjQDwwF7jB3fMs9JSpJ4HzgF3AUHefF8HulYi7Mz09m1FfpjFn5RYa1qzMg+cfz5U9WlI1Pi7a5YmIhFXEQsXM4oDRwFlAJjDHzCa5e2qhNonA/UBvd99iZo0KreIfQDXg5kLtKwCvAr9y92Vm9hBwLfAicC6QGHz1AJ4NXqPC3flq6UZGTUlj/uocmtSuwl8uOoHLT25BlUoKExEpmyJ5pNIdSHf3DAAzGwcMAFILtbkJGO3uWwDcPWv/Anf/0sz6FFlnfWCvuy8Lpj8nFEovBut+zd0dmG1mdcysibv/GP6uHZi788X3WYz6Mo1Fa7fSrE5VHrmkIwNPak7ligoTESnbIhkqzYA1haYz+fmRQzsAM5tBaIhspLt/epB1ZgOVzCzJ3ZOBgUCLg2yvGfCTUDGzYcAwgISEhF/Sn4MqKHAmL1nPqCnpfP/jNhLqVeNvl57IJV2bE1+xQti2IyJSmkUyVKyYeV7M9hOBPkBzYJqZdXT3nOJW6O5uZoOBx82sMvAZkPcLtoe7jwHGACQlJf1s+S+VX+B8vOhHnp6SxrINO2jdoDr/vKwzA7o0pWKcwkREypdIhkom/38UAaHQWFdMm9nungusMLOlhEJmzoFW6u6zgNMAzOxsgqOdEm4vbPLyC/hw4TqempJOxsadtG1UgycHd+GCTk2Jq1BcvomIlH2RDJU5QKKZtQbWAoOBIUXaTASuAF4xswaEAiLjYCs1s0bunhUcqdwLPBIsmgQMD87d9AC2Rup8yszl2fzhvUWs3LSL4xrXZPSQbpzbsTEVFCYiUs5FLFSCy3yHA5MJnS95yd2XBFdsJbv7pGDZ2WaWCuQD97j7JgAzmwYcB9Qws0xClw5PBu4xswuACsCz7j4l2OQnhC4nTid0SfF1kepb7aqVqFGlIv+++iTOOv4YhYmISMBCF0uVT0lJSZ6cnHxY73V3Qh+NEREpX8xsrrsnFbdMZ5IPkwJFROTnFCoiIhI2ChUREQkbhYqIiISNQkVERMJGoSIiImGjUBERkbBRqIiISNiU6w8/mtlGYNVhvr0BobsmlwXqS+lUVvpSVvoB6st+Ld29YXELynWoHAkzSz7QJ0pjjfpSOpWVvpSVfoD6UhIa/hIRkbBRqIiISNgoVA7fmGgXEEbqS+lUVvpSVvoB6ssh6ZyKiIiEjY5UREQkbBQqIiISNgqVEjKzlWa2yMxSzCw5mFfPzD43s7TgtW606yyJA/RlpJmtDealmNl50a7zUMysjpm9Y2Y/mNn3ZtYrhvdJcX2JxX3SvlC9KWa2zcxGxNp+OUg/Ym6fAJjZXWa2xMwWm9lbZlbFzFqb2bfBPhlvZvFh2ZbOqZSMma0Ektw9u9C8vwOb3f1RM7sPqOvu90arxpI6QF9GAjvc/bFo1fVLmdmrwDR3fyH4gagG/IHY3CfF9WUEMbZPCjOzOGAt0AO4nRjcL/CzflxHjO0TM2sGTAc6uPtuM3ub/3/8+nvuPs7MngMWuPuzR7o9HakcmQHAq8H3rwIXR7GWcsXMagGnAy8CuPs+d88hBvfJQfoS634FLHf3VcTgfimkcD9iVUWgqplVJPQHy49AP+CdYHnY9olCpeQc+MzM5prZsGDeMe7+I0Dw2ihq1f0yxfUFYLiZLTSzl0r78ARwLLAReNnM5pvZC2ZWndjcJwfqC8TWPilqMPBW8H0s7pf9CvcDYmyfuPta4DFgNaEw2QrMBXLcPS9olgk0C8f2FCol19vduwHnAreb2enRLugIFNeXZ4E2QBdC//H+GcX6SqIi0A141t27AjuB+6Jb0mE7UF9ibZ/8VzCEdxEwIdq1HIli+hFz+yQIvgFAa6ApUJ3Qz35RYTkXolApIXdfF7xmAe8D3YENZtYEIHjNil6FJVdcX9x9g7vnu3sB8Dyh/pVmmUCmu38bTL9D6BdzLO6TYvsSg/uksHOBee6+IZiOxf0CRfoRo/vkTGCFu29091zgPeAUoE4wHAbQHFgXjo0pVErAzKqbWc393wNnA4uBScC1QbNrgQ+iU2HJHagv+3/gA5cQ6l+p5e7rgTVm1j6Y9SsglRjcJwfqS6ztkyKu4KdDRjG3XwI/6UeM7pPVQE8zq2Zmxv//rEwFBgZtwrZPdPVXCZjZsYT+oofQUMVYd3/EzOoDbwMJhHbcZe6+OUpllshB+vI6oUN6B1YCN+8fAy+tzKwL8AIQD2QQujKnAjG2T+CAfRlFjO0TADOrBqwBjnX3rcG8WPxZKa4fMfdzAmBmfwEuB/KA+cCNhM6hjAPqBfOucve9R7wthYqIiISLhr9ERCRsFCoiIhI2ChUREQkbhYqIiISNQkVERMJGoSIiImGjUBEpZczsFTMbeOiWIqWPQkVERMJGoSJSAmbWKnh41vPBw44+M7OqZvaVmSUFbRoEz6rBzIaa2UQz+9DMVpjZcDO7O7gL8Wwzq1fC7Z5kZl8Hd5SeXOj+WTeZ2RwzW2Bm7wa34KhtoQewVQjaVDOzNWZWyczamNmnwXqmmdlxQZvLggc3LTCzbyLyjyflikJFpOQSgdHufgKQA1x6iPYdgSGEbjr4CLAruAvxLOCaQ23MzCoBTwED3f0k4KVgPRB6uNLJ7t4Z+B64IbiVyALgjKDNhcDk4CaCY4A7gvX8DngmaPMn4JxgPRcdqiaRQ6l46CYiEljh7inB93OBVodoP9XdtwPbzWwr8GEwfxHQqQTba08omD4P3QeQOEK3WwfoaGYPA3WAGsDkYP54Qvd4mkroOSDPmFkNQnelnRCsB6By8DoDeCV4GuB7JahJ5KAUKiIlV/hme/lAVUI36Nt/xF/lIO0LCk0XULKfPQOWuHuvYpa9Alzs7gvMbCjQJ5g/CfhrMLx2EjCF0PMzcty9S9GVuPstZtYDOB9IMbMu7r6pBLWJFEvDXyJHZiWhX97w/7cRD5elQEMz6wWh4TAzOyFYVhP4MRgiu3L/G9x9B/Ad8CTwUfDsj23ACjO7LFiPmVnn4Ps27v6tu/8JyAZahLkPUs4oVESOzGPArWY2E2gQzhW7+z5CQfU3M1sApBAaxgL4I/At8DnwQ5G3jgeuCl73uxK4IVjPEkJPAgT4h5ktMrPFwDeEzsmIHDbd+l5ERMJGRyoiIhI2OlEvEiVmNhroXWT2k+7+cjTqEQkHDX+JiEjYaPhLRETCRqEiIiJho1AREZGwUaiIiEjY/B/25E9brHbMTAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "n_leafs = len(num_leaves_s)\n",
    "\n",
    "x_axis = num_leaves_s\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_means, yerr=test_stds,label = ' Test')\n",
    "#plt.errorbar(x_axis, -train_means, yerr=train_stds,label = ' Train')\n",
    "plt.xlabel( 'num_leaves' )\n",
    "plt.ylabel( 'Log Loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 上图表示50 的时候损失最小，可以将 num_leaves 调得更小尝试下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned_parameters {'num_leaves': range(10, 50, 20)}\n",
      "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done   2 out of   6 | elapsed: 155.9min remaining: 311.9min\n",
      "[Parallel(n_jobs=6)]: Done   3 out of   6 | elapsed: 156.1min remaining: 156.1min\n",
      "[Parallel(n_jobs=6)]: Done   4 out of   6 | elapsed: 174.2min remaining: 87.1min\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed: 174.8min remaining:    0.0s\n",
      "[Parallel(n_jobs=6)]: Done   6 out of   6 | elapsed: 174.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12 s, sys: 3 s, total: 15 s\n",
      "Wall time: 2h 54min 54s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      class_weight=None, colsample_bytree=0.7,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=64,\n",
       "                                      max_depth=7, min_child_samples=20,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=9431,\n",
       "                                      n_jobs=6, num_leaves=31,\n",
       "                                      objective='regression', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.7,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='warn', n_jobs=6, param_grid={'num_leaves': range(10, 50, 20)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=10)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7,\n",
    "          'max_bin': 64, #原始特征为整数，很少超过100\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "num_leaves_s = range(10,50,20) #50,60,70,80\n",
    "tuned_parameters = dict( num_leaves = num_leaves_s)\n",
    "print('tuned_parameters',tuned_parameters)\n",
    "grid_search_num_leaves = GridSearchCV(lg, n_jobs=6, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=10, refit = False)\n",
    "grid_search_num_leaves.fit(train_X , train_Y)\n",
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(model_path + 'grid_search_num_leaves.pkl','wb') as fw:\n",
    "        pk.dump(grid_search_num_leaves,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump grid_search_num_leaves error,',e)\n",
    "try:\n",
    "    # 序列化到磁盘中\n",
    "    with open(model_path + 'lg_2_1_num_leaves_1.pkl','wb') as fw:\n",
    "        pk.dump(lg,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg error,',e)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'num_leaves': 30}"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_num_leaves.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([4602.93072176, 6923.4008619 ]),\n",
       " 'std_fit_time': array([220.89841007, 662.05124328]),\n",
       " 'mean_score_time': array([4436.01588472, 3545.82270972]),\n",
       " 'std_score_time': array([229.89837382, 657.89431326]),\n",
       " 'param_num_leaves': masked_array(data=[10, 30],\n",
       "              mask=[False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'num_leaves': 10}, {'num_leaves': 30}],\n",
       " 'split0_test_score': array([-0.62310613, -0.61718788]),\n",
       " 'split1_test_score': array([-0.62309994, -0.61761465]),\n",
       " 'split2_test_score': array([-0.62278037, -0.61752934]),\n",
       " 'mean_test_score': array([-0.62299548, -0.61744396]),\n",
       " 'std_test_score': array([0.00015212, 0.00018439]),\n",
       " 'rank_test_score': array([2, 1], dtype=int32)}"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_num_leaves.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEHCAYAAAC5u6FsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3hUddrG8e+TQu8SFAGliyCCEFFaUJeuggUR7KhgQym7rrrr7uu6TdfdACqKiBVlRUQBFQ3YEkBaQHpvUqULKEqR5/1jjrsxm0AGMpmU+3Ndc2XmN79z5pnjkTvnzOQ55u6IiIjkVEy0CxARkYJFwSEiImFRcIiISFgUHCIiEhYFh4iIhCUu2gXkhcqVK3vNmjWjXYaISIEyb968Xe6ekHm8SARHzZo1SU9Pj3YZIiIFipl9ndW4TlWJiEhYFBwiIhIWBYeIiIRFwSEiImFRcIiISFgiGhxm1tnMVprZGjN7OJs5Pc1smZktNbMxwVhTM5sZjC0ys+szzH/JzBYG4++YWZlIvgcREfmliAWHmcUCw4EuQEOgt5k1zDSnHvAI0NrdGwEDg6cOArcEY52BoWZWIXhukLs3cffzgY1A/0i9BxER+V+RPOJoAaxx93Xufhh4C+ieaU5fYLi77wVw9x3Bz1Xuvjq4vxXYASQEj/cDmJkBJYGI9YVPW7WT6at3RWr1IiIFUiSDoxqwKcPjzcFYRvWB+mY2w8xmmVnnzCsxsxZAMWBthrFXgG+ABsAzWb24mfUzs3QzS9+5c+dJvYHnvljDTS/N5rfvLGTfD0dOah0iIoVNJIPDshjLfHQQB9QDLgF6A6MynJLCzKoCo4E+7n7sPytx7wOcCSwHricL7j7S3RPdPTEh4X/+Yj5HXu3Tgrvb1WH8/C10SE4lZek3J7UeEZHCJJLBsRmokeFxdWBrFnMmuvsRd18PrCQUJJhZOeBD4FF3n5V55e7+EzAWuDYCtQNQIj6Wh7s0YMK9rTmtTHHuGj2P+96cz84DhyL1kiIi+V4kg2MuUM/MaplZMaAXMCnTnAnApQBmVpnQqat1wfz3gNfdfdzPky2k7s/3gSuBFRF8DwA0rl6eSf1b82Cnc5i6bDvtk1MZP28zuuyuiBRFEQsOdz9K6BtPKYROKb3t7kvN7HEz6xZMSwF2m9ky4HPgQXffDfQEkoDbzGxBcGtK6PTXa2a2GFgMVAUej9R7yCg+Nob7Lq3L5AFtqJNQml+PW8htr8xly7c/5MXLi4jkG1YUfmtOTEz03OyO+9MxZ/TMDfwjZSUGPNSlATdddDYxMVl9rCMiUjCZ2Tx3T8w8rr8cPwmxMcZtrWuRMjCJZmdX5I8Tl3L9yJms3fldtEsTEYk4BccpqFGpFK/f3oKnepzPym8O0GXYNJ77Yg1Hfjp24oVFRAooBccpMjOuS6zBJ79ux2XnVOEfH6/kquEzWLJlX7RLExGJCAVHLqlStgQjbm7O8zc2Y/v+Q3QfPoOnUlbw45Gfol2aiEiuUnDksi6Nq/LJ4CSualqN4Z+vpevT00jfsCfaZYmI5BoFRwRUKFWMf/Vswmu3t+DQkWNc98JMHpu0lO8PHY12aSIip0zBEUHt6ieQMiiJW1vW5LWZG+g4JI20VSfXN0tEJL9QcERYmeJxPNatEePuaknx+BhueXkOvxm3kG8PHo52aSIiJ0XBkUcSa1Zi8gNtue/SOrz31RbaJ6fx0eJt0S5LRCRsCo48VCI+lgc7NWBS/9acXq4497w5n7tHz2PH/h+jXZqISI4pOKKg0ZnlmXBfa37b+Rw+W7mD9smpjEvfpKaJIlIgKDiiJD42hnsvqctHA9pyzhllefCdRdzy8hw27TkY7dJERI5LwRFldRLKMLZfS/7cvRHzv95Lp6FpvDpjPceO6ehDRPInBUc+EBNj3NyyJimDkriwZiUee38Z170wkzU7DkS7NBGR/6HgyEeqVyzFq30uJLlnE9bu/I6uw6bz7Ger1TRRRPIVBUc+Y2Zc06w6Uwe1o0PD0/nnlFV0e1ZNE0Uk/1Bw5FMJZYsz/MZmjLipObu+CzVNfOIjNU0UkehTcORznc87g08GtaNHs+qMSF1L12HTmLNeTRNFJHoUHAVA+VLxPNnjfN644yIO/3SMni/M5A8TlvCdmiaKSBQoOAqQNvUqM2VQEre3rsUbs7+mY3Iqn6/cEe2yRKSIUXAUMKWKxfHHKxvyzt2tKFU8jj6vzGXw2AXs/V5NE0Ukbyg4CqjmZ1fkwwfa8MBldZm0cCvtk1P5YNFWtS0RkYhTcBRgxeNiGdzxHN6/vw1nVihJ/zFfcdfoeWxX00QRiSAFRyFwbtVyvHdvKx7p0oDUVTtpn5zK2LkbdfQhIhGh4Cgk4mJjuKtdHT4emMS5Vcvx0PjF3PTSbDbuVtNEEcldCo5Cplbl0rzV92L+ctV5LNy0j05D03hp+np+UtNEEcklCo5CKCbGuOnis5kyKImLa1fizx8s49rnv2TVdjVNFJFTF9HgMLPOZrbSzNaY2cPZzOlpZsvMbKmZjQnGmprZzGBskZldn2H+m8E6l5jZy2YWH8n3UJCdWaEkL992IUOvb8rXu7/n8qen8fSnqzl8VE0TReTkRSw4zCwWGA50ARoCvc2sYaY59YBHgNbu3ggYGDx1ELglGOsMDDWzCsFzbwINgMZASeDOSL2HwsDMuOqCakwd3I7O51Uleeoquj07nYWbvo12aSJSQEXyiKMFsMbd17n7YeAtoHumOX2B4e6+F8DddwQ/V7n76uD+VmAHkBA8nuwBYA5QPYLvodCoXKY4z/S+gBdvSWTvwcNc/dwM/j55OT8cVtNEEQlPJIOjGrApw+PNwVhG9YH6ZjbDzGaZWefMKzGzFkAxYG2m8XjgZuDjrF7czPqZWbqZpe/cufMU3kbh0qHh6Uwd3I7rL6zBC2nr6DIsjZlrd0e7LBEpQCIZHJbFWOav9sQB9YBLgN7AqAynpDCzqsBooI+7Zz4x/xyQ5u7Tsnpxdx/p7onunpiQkHCSb6FwKlcinr9fcz5j7ryIYw69X5zF795bzP4fj0S7NBEpACIZHJuBGhkeVwe2ZjFnorsfcff1wEpCQYKZlQM+BB5191kZFzKz/yN06mpwhGovElrVrUzKwCTubFOLt+ZspGNyGp+t2B7tskQkn4tkcMwF6plZLTMrBvQCJmWaMwG4FMDMKhM6dbUumP8e8Lq7j8u4gJndCXQCemdxFCJhKlkslkevaMj4e1pRrmQct7+azoC3vmL3d4eiXZqI5FMRCw53Pwr0B1KA5cDb7r7UzB43s27BtBRgt5ktAz4HHnT33UBPIAm4zcwWBLemwTIjgNOBmcH4HyP1HoqSC86qyAf3t2Vg+3pMXryNDkPSmLRQTRNF5H9ZUfiHITEx0dPT06NdRoGx8psD/Hb8IhZu+pb251bhL1c15ozyJaJdlojkMTOb5+6Jmcf1l+PyP845oyzv3tOKRy8/l+lrdtEhOZUxszdyTG1LRAQFh2QjNsa4s21tUgYmcV618vzuvcXcMGoWG3Z9H+3SRCTKFBxyXGefVpoxfS/i79c0ZumW/XQelsaLaevUNFGkCFNwyAmZGb1bnMXUwe1oU7cyf528nGuem8HKb9Q0UaQoUnBIjp1RvgQv3pLIM70vYPPeH7jimWkMmbpKTRNFihgFh4TFzLiyyZlMHdyOyxtXZdinq7nimWksUNNEkSJDwSEnpVLpYgztdQEv35bIgR+Pcs1zM/jLB8s4ePhotEsTkQhTcMgpuazB6UwZlETvFmcxavp6Og+dxpdrdkW7LBGJIAWHnLKyJeL569WNeavfxcQY3DBqNg+PX8S+H9Q0UaQwUnBIrrm49ml8NCCJu5Jq83b6JjoOSWXqMjVNFClsFBySq0oWi+WRrucy4b7WVCxVjL6vp9N/zHx2qWmiSKGh4JCIOL96BSb1b8OvO9RnytLtdEhOZcJXW9Q0UaQQUHBIxBSLi+H+X9XjwwfaULNyaQaOXcDtr85l67c/RLs0ETkFCg6JuHqnl+Wdu1vxxysaMmvdHjoOSWP0rK/VNFGkgFJwSJ6IjTFub1OLKYOSaFqjAn+YsIReL85ivZomihQ4Cg7JUzUqlWL0HS34x7Xns3zbfjoPTWNE6lqO/qS2JSIFhYJD8pyZ0fPCGnwyuB3t6ifwxEcruPq5L1m2dX+0SxORHFBwSNScXq4EL9zcnOE3NGPbvh/o9ux0/jVlJYeO/hTt0kTkOBQcElVmxuXnV2XqoHZ0a3omz3y2hsufns68r/dGuzQRyYaCQ/KFiqWLkdyzKa/0uZCDh47SY8SX/On9pXx/SE0TRfIbBYfkK5eeU4Upg9tx88Vn88qMDXQamsa01TujXZaIZKDgkHynTPE4Hu9+Hm/f1ZJisTHc/NIcfvvOQvYdVNNEkfxAwSH5VotalZg8oC33XFKH8fO30H5IKh8v+SbaZYkUeQoOyddKxMfyUOcGTLyvNQllinP3G/O478357Dygpoki0aLgkALhvGrlmdi/NQ92Ooepy7bTPjmV8fM2q2miSBQoOKTAiI+N4b5L6zJ5QFvqVinDr8ct5NZX5rJ578FolyZSpEQ0OMyss5mtNLM1ZvZwNnN6mtkyM1tqZmOCsaZmNjMYW2Rm12eY3z9Yn5tZ5UjWL/lT3SplGHdXS/7UrRHpG/bQaUgar8/coKaJInnEInWob2axwCqgA7AZmAv0dvdlGebUA94GLnP3vWZWxd13mFl9wN19tZmdCcwDznX3b83sAmAv8AWQ6O4nvMB1YmKip6en5/ZblHxg056D/O69xUxbvYsLa1bkiWvPp05CmWiXJVIomNk8d0/MPB7JI44WwBp3X+fuh4G3gO6Z5vQFhrv7XgB33xH8XOXuq4P7W4EdQELw+Ct33xDBuqUAqVGpFK/f3oJ/XteEVdu/o8uwaTz3xRqOqGmiSMREMjiqAZsyPN4cjGVUH6hvZjPMbJaZdc68EjNrARQD1obz4mbWz8zSzSx95079AVlhZmb0aF6dqYOT+FWDKvzj45VcNXwGS7bsi3ZpIoVSJIPDshjLfF4sDqgHXAL0BkaZWYX/rMCsKjAa6OPuYf0K6e4j3T3R3RMTEhLCKlwKpiplS/D8Tc15/sZmbN9/iO7DZ/CPj1fw4xE1TRTJTZEMjs1AjQyPqwNbs5gz0d2PuPt6YCWhIMHMygEfAo+6+6wI1imFTJfGVflkcBLXXFCN575YS9enp5G+YU+0yxIpNCIZHHOBemZWy8yKAb2ASZnmTAAuBQi+IVUfWBfMfw943d3HRbBGKaQqlCrGU9c14fXbW3DoyDGue2Em/zdxCd+paaLIKYtYcLj7UaA/kAIsB95296Vm9riZdQumpQC7zWwZ8DnwoLvvBnoCScBtZrYguDUFMLMHzGwzoSOYRWY2KlLvQQq+pPoJTBmUxK0ta/L6rK/pNCSN1FX6zEvkVETs67j5ib6OKwDpG/bw0PhFrN35Pdc2q84frjiXCqWKRbsskXwrGl/HFclXEmtW4sMH2tL/0rpMWLCF9smpTF68LdpliRQ4Cg4pUkrEx/KbTucwqX9rTi9XgnvfnM/do+exY/+P0S5NpMA4YXCYWR0zKx7cvyT4jKHCiZYTyc8anVmeife15qHODfhs5Q7aJ6fydvomNU0UyYGcHHGMB34ys7rAS0AtYExEqxLJA3GxMdxzSR0+HtCWBmeU47fvLOKWl+ewaY+aJoocT06C41jwDamrgaHuPgioGtmyRPJO7YQyvNXvYv7cvRHzv95Lp6FpvDJjPT+paaJIlnISHEfMrDdwK/BBMBYfuZJE8l5MjHFzy5pMGdyOC2tW4k/vL+O6EV+yZseBaJcmku/kJDj6AC2Bv7r7ejOrBbwR2bJEoqNahZK82udCkns2Yd2u7+k6bDrPfrZaTRNFMgjr7zjMrCJQw90XRa6k3Ke/45CTsfPAIR57fykfLtpGgzPK8lSPJjSuXj7aZYnkmZP+Ow4z+8LMyplZJWAh8IqZJUeiSJH8JKFscYbf0IwXbm7Onu8Pc9VzM3jiIzVNFMnJqary7r4fuAZ4xd2bA+0jW5ZI/tGp0RlMHdyOHs2qMyJ1LV2GTWP2ut3RLkskanISHHFBe/Oe/PfDcZEipXzJeJ7scT5v3nkRR48d4/qRs/jDhCUc+PFItEsTyXM5CY7HCTUjXOvuc82sNrA6smWJ5E+t61YmZWASt7euxRuzQ00TP1+xI9plieQpNTkUOUnzN+7loXcWsXrHd1x9QTX+cEVDKpVW00QpPE7lw/HqZvaeme0ws+1mNt7MqkemTJGCo9lZFfnggTY88Kt6vL9wKx2SU/lg0Va1LZFCLyenql4hdAGmMwldM/z9YEykyCseF8vgDvV5//42VKtYkv5jvqLf6HlsV9NEKcRyEhwJ7v6Kux8Nbq8Cuoi3SAbnVi3Hu/e04nddG5C2aiftk1MZO3ejjj6kUMpJcOwys5vMLDa43QTou4gimcTFxtAvqQ4pA5NoWLUcD41fzI2jZrNxt5omSuGSk+C4ndBXcb8BtgE9CLUhEZEs1Kxcmn/3vZi/Xn0eizbvo+PQVEZNW6emiVJonDA43H2ju3dz9wR3r+LuVxH6Y0ARyUZMjHHjRWczdXASrepU5i8fLufa579k1XY1TZSC72SvADg4V6sQKaSqli/JS7cmMqxXUzbuOcjlT09j2CerOXxUTROl4DrZ4LBcrUKkEDMzujetxtRBSXQ5rypDPllFt2ens3DTt9EuTeSknGxw6GStSJhOK1Ocp3tfwKhbEvn24BGufm4Gf5u8nB8Oq2miFCxx2T1hZgfIOiAMKBmxikQKufYNT6dF7Ur8ffIKRqatI2XpNzxxzfm0rHNatEsTyZFsjzjcvay7l8viVtbdsw0cETmxciXi+fs1jRnT9yIAer84i0feXcx+NU2UAuBkT1WJSC5oVacyHw9Iom/bWoydu5GOyWl8unx7tMsSOS4Fh0iUlSwWy+8vb8i797amfMl47ngtnQf+/RW7vzsU7dJEsqTgEMknmtaowPv3t2FQ+/p8tGQbHYakMXHBFrUtkXwnosFhZp3NbKWZrTGzh7OZ09PMlpnZUjMbE4w1NbOZwdgiM7s+w/xaZjbbzFab2VgzUx9rKTSKxcUwoH09Pri/LTUqlWLAWwu487V0tu37IdqlifxHTtqqHzCz/Zlum4JW67WPs1wsMBzoAjQEeptZw0xz6gGPAK3dvREwMHjqIHBLMNYZGGpmFYLnngSGuHs9YC9wR1jvWKQAOOeMsrx7TysevfxcZqzdRcfkNMbM3sgxtS2RfCAnRxzJwIOEWqpXB34DvAi8Bbx8nOVaAGvcfZ27Hw7md880py8w3N33Arj7juDnKndfHdzfCuwAEszMgMuAd4LlXwOuysF7EClwYmOMO9vWJmVgEo2rl+d37y3mhlGz2LDr+2iXJkVcToKjs7u/4O4H3H2/u48Eurr7WKDicZarBmzK8HhzMJZRfaC+mc0ws1lm1jnzSsysBVAMWAucBnzr7kePs86fl+tnZulmlr5z584cvE2R/Ons00rz5p0X8cQ1jVm6ZT+dhqYxMm0tR39S2xKJjpwEx7Hgc4iY4NYzw3PHO27Oqi1J5vlxQD3gEqA3MCrDKSnMrCowGujj7sdyuM7QoPtId09098SEBF0+RAo2M6NXi7OYOrgdbesl8LfJK7j2+S9Z8c3+aJcmRVBOguNG4GZCp4t2BPdvMrOSQP/jLLcZqJHhcXVgaxZzJrr7EXdfD6wkFCSYWTngQ+BRd58VzN8FVDCzuOOsU6TQOqN8CV68pTnP9L6AzXt/4Iqnp5M8dRWHjqptieSdnLRVX+fuV7p75eB2pbuvcfcf3H36cRadC9QLvgVVDOhF6BK0GU0ALgUws8qETl2tC+a/B7zu7uMy1OLA54SuCQJwKzAxZ29VpHAwM65sciZTB7fjyiZn8vSnq7nymel8tXFvtEuTIiIn36qqHnyDaoeZbTez8WZW/UTLBZ9D9AdSgOXA2+6+1MweN7NuwbQUYLeZLSMUCA+6+25CF45KAm4zswXBrWmwzEPAYDNbQ+gzj5fCfM8ihUKl0sUYcn1TXr4tkQM/HuWa57/kzx8s4+DhoydeWOQU2In+uMjMpgJjCH3WAHATcKO7d4hwbbkmMTHR09PTo12GSMQc+PEIT368gjdmbaRGpZI8cc35tK5bOdplSQFnZvPcPTHzeE4+40hw91fc/WhwexXQp80i+UjZEvH85arGjO13MXExMdw4ajYPj1/Evh/UNFFyX06CY5eZ3WRmscHtJmB3pAsTkfBdVPs0PhrQlrva1ebt9E10SE5lytJvol2WFDI5CY7bCX3m8A2wjdAH030iWZSInLwS8bE80uVcJtzXmkqli9Fv9Dz6j5nPLjVNlFySk29VbXT3bu6e4O5V3P0q4Jo8qE1ETsH51UNNE3/doT5Tlm6nfXIq7321WU0T5ZSdbJPDwblahYhERHxsDPf/qh4fPtCGWpVLM2jsQvq8Opct36ppopy8kw2OrP6CW0TyqXqnl+Wdu1vxf1c2ZPa6PXRMTmX0rK/VNFFOyskGh/Y2kQImNsbo07oWUwYlccFZFfnDhCX0GjmLdTu/i3ZpUsBkGxzZtFPfb2YHgDPzsEYRyUU1KpVi9B0t+EeP81nxzX66DJvGiFQ1TZScyzY43L2su5fL4lbW3eOyW05E8j8zo2diDT4Z3I5LzkngiY9WcNVzM1i2VU0T5cR06ViRIqxKuRKMuKk5z93YjG/2/Ui3Z6fzrykr1TRRjkvBIVLEmRldG1dl6qB2dG9ajWc+W0PXYdOY9/WeaJcm+ZSCQ0QAqFi6GP/q2YTXbm/Bj0eO0WPETB6btJTvD6lpovySgkNEfqFd/QRSBiVxy8Vn8+qXG+g0NI1pq3UVTfkvBYeI/I8yxeP4U/fzGHd3S4rFxXDzS3N4cNxC9h1U00RRcIjIcVxYsxKTH2jLvZfU4d2vttB+SCofL1HTxKJOwSEix1UiPpbfdm7AxPtak1CmOHe/MY9735zHjgM/Rrs0iRIFh4jkyHnVyjOxf2se7HQOnyzfQYfkNN6Zp6aJRZGCQ0RyLD42hvsurcvkB9pSr0oZfjNuIbe+MpfNew9GuzTJQwoOEQlb3SplePuulvypWyPSN+yh45A0Xvtyg5omFhEKDhE5KTExxq2tajJlUBKJNSvxf5OW0vOFmaxV08RCT8EhIqekesVSvNbnQv55XRNW7/iOLsOmMfzzNRxR08RCS8EhIqfMzOjRvDpTByfR/twqPJWyku7PzmDJln3RLk0iQMEhIrmmStkSPHdjc0bc1Iyd3x2i+/AZPPnxCn48oqaJhYmCQ0RyXefzqvLJoHZcc0E1nv9iLV2HTWPuBjVNLCwUHCISEeVLxfPUdU0YfUcLDv90jOtGzOSPE5fwnZomFngKDhGJqLb1EkgZmMRtrWoyetbXdBqSRuoqNU0syCIaHGbW2cxWmtkaM3s4mzk9zWyZmS01szEZxj82s2/N7INM8y8zs/lmtsTMXjMzXY1QJJ8rXTyOx7o14p27W1IiPoZbX57D4LcXsPf7w9EuTU5CxILDzGKB4UAXoCHQ28waZppTD3gEaO3ujYCBGZ5+Crg50/wY4DWgl7ufB3wN3Bqp9yAiuav52ZX48IG23H9ZXSYt2EqHIalMXrxNbUsKmEgecbQA1rj7Onc/DLwFdM80py8w3N33Arj7jp+fcPdPgQOZ5p8GHHL3VcHjqcC1kSheRCKjRHwsv+54DpP6t6Fq+ZLc++Z87n5jHjv2q2liQRHJ4KgGbMrweHMwllF9oL6ZzTCzWWbW+QTr3AXEm1li8LgHUCNXqhWRPNXwzHK8d28rHu7SgC9W7qR9cipvp2/S0UcBEMngsCzGMu8RcUA94BKgNzDKzCpkt0IP7VG9gCFmNofQEUmWX9Ews35mlm5m6Tt36oM4kfwoLjaGu9vV4aMBbWlwRjl++84ibn5pDpv2qGlifhbJ4NjML48GqgNbs5gz0d2PuPt6YCWhIMmWu89097bu3gJIA1ZnM2+kuye6e2JCQsJJvwkRibzaCWV4q9/F/Pmq8/hq4146DknjlRnr+UlNE/OlSAbHXKCemdUys2KEjhQmZZozAbgUwMwqEzp1te54KzWzKsHP4sBDwIhcrltEoiAmxrj54rOZMrgdF9WuxJ/eX8Z1I75k9fbMH3VKtEUsONz9KNAfSAGWA2+7+1Ize9zMugXTUoDdZrYM+Bx40N13A5jZNGAc8Csz22xmnYJlHjSz5cAi4H13/yxS70FE8l61CiV55bYLGXp9U9bv+p7Ln57OM5+uVtPEfMSKwgdRiYmJnp6eHu0yRCRMu747xGOTlvLBom00OKMsT/VoQuPq5aNdVpFhZvPcPTHzuP5yXETyrcplivPsDc0YeXNz9nx/mO7Dp/P3j5araWKUKThEJN/r2OgMpg5uR8/EGryQuo4uw6Yxe93uaJdVZCk4RKRAKF8ynieuPZ8377yIo8eOcf3IWTw6YTEHfjwS7dKKHAWHiBQoretWJmVgEne2qcWY2RvpOCSNz1fsOPGCkmsUHCJS4JQqFsejVzRk/D2tKFM8jj6vzmXgW1+xR00T84SCQ0QKrAvOqsgHD7RhwK/q8cGibXRITuX9hVvVtiTCFBwiUqAVj4tlUIf6fPBAG6pVLMn9//6Kvq/PY7uaJkaMgkNECoUGZ5Tj3Xta8fuu5zJtdahp4ltzNuroIwIUHCJSaMTFxtA3qTYpA5NoWLUcD7+7mBtHzWbjbjVNzE0KDhEpdGpWLs2/+17M365uzOLN++g4NJVR09apaWIuUXCISKEUE2PccNFZTBmcROs6lfnLh8u55vkvWfmNmiaeKgWHiBRqVcuXZNStiQzr1ZRNew5yxTPTGPrJKg4fVdPEk6XgEJFCz8zo3rQaUwcl0bVxVYZ+sporn5nOwk3fRru0AknBISJFxmllijOs1wWMuiWRfT8c4ernZvDXD5fxw2E1TQyHgkNEipz2DU9nyuAkerU4ixenrafzsDS+XLsr2mUVGAoOESmSypWI529XN+bffS8G4IYXZ/PIu4vZr6aJJ6TgEMHi/VoAAA3KSURBVJEirWWd0/h4QBL9kmozdu5GOiSn8smy7dEuK19TcIhIkVeyWCy/63ou793bmoqlinHn6+k88O+v2P3doWiXli8pOEREAk1qVGBS/zYMal+fj5Zso31yKhMXbFHbkkwUHCIiGRSLi2FA+3p8+EBbzj6tNAPeWsCdr6Wzbd8P0S4t31BwiIhkof7pZRl/Tyv+cEVDvly7mw7Jabw5+2uOqW2JgkNEJDuxMcYdbWqRMjCJJjXK8/v3ltD7xVms3/V9tEuLKgWHiMgJnHVaKd644yKevLYxy7btp/PQNEamreXoT0WzbYmCQ0QkB8yM6y88i08GtyOpfgJ/m7yCa57/kuXb9ke7tDyn4BARCcPp5Uow8ubmPHvDBWzZ+wNXPjOd5KmrOHS06LQtUXCIiITJzLji/DP5ZHA7ujU5k6c/Xc0VT09n/sa90S4tTyg4REROUsXSxUi+vimv9LmQ7w8d5drnv+TPHyzj4OGj0S4toiIaHGbW2cxWmtkaM3s4mzk9zWyZmS01szEZxj82s2/N7INM839lZvPNbIGZTTezupF8DyIiJ3LpOVVIGZTETRedzUvT19NpaBoz1hTepokRCw4ziwWGA12AhkBvM2uYaU494BGgtbs3AgZmePop4OYsVv08cKO7NwXGAI9GoHwRkbCULRHPn686j7H9LiYuJoYbR83moXcWse+Hwtc0MZJHHC2ANe6+zt0PA28B3TPN6QsMd/e9AO6+4+cn3P1TIKtrPDpQLrhfHtia24WLiJysi2qfxkcD2nJ3uzq8M38zHZJTmbL0m2iXlasiGRzVgE0ZHm8OxjKqD9Q3sxlmNsvMOudgvXcCk81sM6EjkieymmRm/cws3czSd+7ceRLli4icnBLxsTzcpQET7m3NaWWK02/0PO4bM5+dBwpH08RIBodlMZb5b/XjgHrAJUBvYJSZVTjBegcBXd29OvAKkJzVJHcf6e6J7p6YkJAQVuEiIrmhcfXyTOrfmt90rM/UpdvpMCSVd+dvLvBNEyMZHJuBGhkeV+d/TyttBia6+xF3Xw+sJBQkWTKzBKCJu88OhsYCrXKvZBGR3BUfG0P/y+oxeUAbalcuzeC3F9Ln1bls+bbgNk2MZHDMBeqZWS0zKwb0AiZlmjMBuBTAzCoTOnW17jjr3AuUN7P6weMOwPJcrVpEJALqVinLuLtb8diVDZmzfg8dk1MZPXNDgWyaGLHgcPejQH8ghdA/7m+7+1Ize9zMugXTUoDdZrYM+Bx40N13A5jZNGAc8Csz22xmnYJ19gXGm9lCQp9xPBip9yAikptiY4zbWoeaJjY7uyJ/mLiUXiNnsW7nd9EuLSxW0M+15URiYqKnp6dHuwwRkf9wd96Zt5k/f7CMH48eY1D7+vRtW4u42Pzzd9lmNs/dEzOP558KRUSKEDPjusQafDK4HZedU4UnP17BVc/NYNnW/N80UcEhIhJFVcqVYMTNzXn+xmZ8s+8Q3Z6dzj9TVvLjkfzbNFHBISKSD3RpXJVPBifRvWk1nv18DZc/PY15X++JdllZUnCIiOQTFUoV4189m/Da7S348cgxeoyYyWOTlvL9ofzVNFHBISKSz7Srn0DKoCRuufhsXpu5gY5D0khblX86YCg4RETyoTLF4/hT9/N4+66WFI+P4ZaX5/CbcQvZdzD6TRMVHCIi+diFNSsx+YG23HdpHd77agvth6Ty8ZJtUa1JwSEiks+ViI/lwU4NmNS/NVXKFufuN+Zzzxvz2HHgx6jUo+AQESkgGp1Zngn3tea3nc/h0xU76JCcxrj0TXneNFHBISJSgMTHxnDvJXX5aEBb6p9ehgffWcQtL89h056DeVaDgkNEpACqk1CGsf1a8nj3Rsz/ei+dhqbx6oz1edI0UcEhIlJAxcQYt7SsScqgJBJrVuKx95fR84WZrNkR2aaJCg4RkQKuesVSvNbnQv51XRPW7PyOrsOmMfzzNRz56VhEXk/BISJSCJgZ1zavztRB7ejQ8HSeSllJ92dnsH7X97n+WgoOEZFCJKFscYbf2IwRNzUHoELJ+Fx/jbhcX6OIiERd5/POoFOj0zGzXF+3jjhERAqpSIQGKDhERCRMCg4REQmLgkNERMKi4BARkbAoOEREJCwKDhERCYuCQ0REwmJ53cc9GsxsJ/D1SS5eGdiVi+XkFtUVHtUVHtUVnsJa19nunpB5sEgEx6kws3R3T4x2HZmprvCorvCorvAUtbp0qkpERMKi4BARkbAoOE5sZLQLyIbqCo/qCo/qCk+RqkufcYiISFh0xCEiImFRcIiISFiKbHCY2ctmtsPMlmQYq2RmU81sdfCzYjbL3hrMWW1mt+ZBXU+Z2QozW2Rm75lZhWyW3WBmi81sgZml50Fdj5nZluD1FphZ12yW7WxmK81sjZk9nAd1jc1Q0wYzW5DNspHcXjXM7HMzW25mS81sQDAe1X3sOHVFdR87Tl1R3ceOU1dU9zEzK2Fmc8xsYVDXn4LxWmY2O9hvxppZsWyWfyTYVivNrFPYBbh7kbwBSUAzYEmGsX8ADwf3HwaezGK5SsC64GfF4H7FCNfVEYgL7j+ZVV3BcxuAynm4vR4DfnOC5WKBtUBtoBiwEGgYyboyPf8v4I9R2F5VgWbB/bLAKqBhtPex49QV1X3sOHVFdR/Lrq5o72OAAWWC+/HAbOBi4G2gVzA+Argni2UbBtuoOFAr2Hax4bx+kT3icPc0YE+m4e7Aa8H914Crsli0EzDV3fe4+15gKtA5knW5+xR3Pxo8nAVUz63XO5W6cqgFsMbd17n7YeAtQts54nWZmQE9gX/n1uvllLtvc/f5wf0DwHKgGlHex7KrK9r72HG2V05EbB87UV3R2sc85LvgYXxwc+Ay4J1gPLv9qzvwlrsfcvf1wBpC2zDHimxwZON0d98GoR0GqJLFnGrApgyPN5PzHTw33A58lM1zDkwxs3lm1i+P6ukfnN54OZvTLtHcXm2B7e6+Opvn82R7mVlN4AJCvxXmm30sU10ZRXUfy6KufLGPZbO9oraPmVlscIpsB6FfLtYC32b4BSC77XDK20vBEb6sLuKbJ99pNrPfA0eBN7OZ0trdmwFdgPvMLCnCJT0P1AGaAtsIHbJnFrXtBfTm+L8JRnx7mVkZYDww0N3353SxLMZydZtlV1e097Es6soX+9hx/jtGbR9z95/cvSmho8MWwLlZTcti7JS3l4Ljl7abWVWA4OeOLOZsBmpkeFwd2BrpwoIPSK8AbvTgRGVm7r41+LkDeI8wDz/D5e7bg533GPBiNq8Xre0VB1wDjM1uTqS3l5nFE/rH5k13fzcYjvo+lk1dUd/HsqorP+xjx9leUd/HgnV/C3xB6DOOCkFdkP12OOXtpeD4pUnAz99guRWYmMWcFKCjmVUMDps7BmMRY2adgYeAbu5+MJs5pc2s7M/3g7qWZDU3F+uqmuHh1dm83lygXvBtj2JAL0LbOdLaAyvcfXNWT0Z6ewXnvl8Clrt7coanorqPZVdXtPex49QV1X3sOP8dIYr7mJklWPDNNzMrGdSyHPgc6BFMy27/mgT0MrPiZlYLqAfMCauA3P60v6DcCB1ebgOOEErgO4DTgE+B1cHPSsHcRGBUhmVvJ/SB0hqgTx7UtYbQOckFwW1EMPdMYHJwvzahb0osBJYCv8+DukYDi4FFwc5YNXNdweOuhL6NsjYv6grGXwXuzjQ3L7dXG0KH/4sy/HfrGu197Dh1RXUfO05dUd3Hsqsr2vsYcD7wVVDXEoJvdQWvOSf47zkOKB6MdwMez7D874NttRLoEu7rq+WIiIiERaeqREQkLAoOEREJi4JDRETCouAQEZGwKDhERCQsCg4REQmLgkMkCszsVTPrceKZIvmPgkNERMKi4BAJmFnN4II9LwYXx5liZiXN7AszSwzmVDazDcH928xsgpm9b2brzay/mQ02s6/MbJaZVcrh6zY3s9Sgg2pKhl5Wfc1sbnCxnvFmVsrMylvo4kAxwZxSZrbJzOLNrI6ZfRysZ5qZNQjmXGdmS4L1pEVk40mRouAQ+aV6wHB3bwR8C1x7gvnnATcQal73V+Cgu18AzARuOdGLBQ30ngF6uHtz4OVgPQDvuvuF7t6EUB+iO9x9H6EWFu2COVcCKe5+BBgJ3B+s5zfAc8GcPwKdgvV0O1FNIicSd+IpIkXKenf/+TKg84CaJ5j/uYcu8HPAzPYB7wfjiwn1EzqRcwiFz9RQPz1iCfXeAjjPzP4CVADK8N9Gh2OB6wk1tOsFPBe0/W4FjAvWA6ErvAHMAF41s7eB/3R3FTlZCg6RXzqU4f5PQElC16f4+ei8xHHmH8vw+Bg5+//LgKXu3jKL514FrnL3hWZ2G3BJMD4J+HtwKqw58BlQmtBFfJpmXom7321mFwGXAwvMrKm7785BbSJZ0qkqkRPbQOgfaPhvy+rcshJIMLOWEDp1ZWaNgufKAtuC01k3/ryAhy4ZOgcYBnzgoetV7AfWm9l1wXrMzJoE9+u4+2x3/yOwi19ei0EkbAoOkRP7J3CPmX0JVM7NFXvoGtk9gCfNbCGhtt2tgqf/QOgypVOBFZkWHQvcxC8vInQjcEewnqX897rbT5nZYjNbAqQR+oxE5KSprbqIiIRFRxwiIhIWfTguEkFmNhxonWl4mLu/Eo16RHKDTlWJiEhYdKpKRETCouAQEZGwKDhERCQsCg4REQnL/wPCiORo0A+ezgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search_num_leaves.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search_num_leaves.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "n_leafs = len(num_leaves_s)\n",
    "\n",
    "x_axis = num_leaves_s\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_means, yerr=test_stds,label = ' Test')\n",
    "#plt.errorbar(x_axis, -train_means, yerr=train_stds,label = ' Train')\n",
    "plt.xlabel( 'num_leaves' )\n",
    "plt.ylabel( 'Log Loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 综上，num_leaves 取 30 log logss 较低"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. min_child_samples\n",
    "叶子节点的最小样本数目\n",
    "# 下面的这段分析是什么意思？？？？？？\n",
    "叶子节点数目：70，共9类，平均每类8个叶子节点\n",
    "每棵树的样本数目数目最少的类（稀有事件）的样本数目：200 * 2/3 * 0.7 = 100\n",
    "所以每个叶子节点约100/8 = 12个样本点\n",
    "\n",
    "搜索范围：10-50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 下面我添加 early_stopping_round 报错了，原因是什么（没规定验证集？？？）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done   4 out of  12 | elapsed: 177.0min remaining: 354.0min\n",
      "[Parallel(n_jobs=6)]: Done   7 out of  12 | elapsed: 259.8min remaining: 185.5min\n",
      "[Parallel(n_jobs=6)]: Done  10 out of  12 | elapsed: 311.8min remaining: 62.4min\n",
      "[Parallel(n_jobs=6)]: Done  12 out of  12 | elapsed: 321.8min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      class_weight=None, colsample_bytree=0.7,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=64,\n",
       "                                      max_depth=7, min_child_samples=20,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=9431,\n",
       "                                      n_jobs=6, num_leaves=30,\n",
       "                                      objective='regression', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.7,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='warn', n_jobs=6,\n",
       "             param_grid={'min_child_samples': range(10, 90, 20)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':30,\n",
    "          'max_bin': 64, # 稀疏离散特征一共有167维，先不设置很大，提高训练速度\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "#           'early_stopping_round':10, # 添加 early_stopping_round，减少迭代次数\n",
    "#           'metric':'auc'\n",
    "         }\n",
    "lg_v1 = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "min_child_samples_s = range(10,90,20) \n",
    "tuned_parameters = dict( min_child_samples = min_child_samples_s)\n",
    "\n",
    "grid_search_min_child_samples_v1 = GridSearchCV(lg_v1, n_jobs=6,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search_min_child_samples_v1.fit(train_X , train_Y)\n",
    "# grid_search_v1.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(model_path + 'grid_search_min_child_samples_v1.pkl','wb') as fw:\n",
    "        pk.dump(grid_search_min_child_samples_v1,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump grid_search_min_child_samples_v1 error,',e)\n",
    "try:\n",
    "    # 序列化到磁盘中\n",
    "    with open(model_path + 'lg_v1.pkl','wb') as fw:\n",
    "        pk.dump(lg_v1,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg_v1 error,',e)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6171816918134087\n",
      "{'min_child_samples': 70}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search_min_child_samples_v1.best_score_)\n",
    "print(grid_search_min_child_samples_v1.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.61735878, -0.6174219 , -0.61730002, -0.61718169])"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f73b6bd0e80>]"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD4CAYAAAAQP7oXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhU5d3/8fd3Jiv7voNsCci+BAhFFFAQH6u4sVpkE2irbbV92urv6VNbfbppVbSiEhARZBGrotUWRERxIZBEcSEQCIsSUMK+B7Lcvz8ytGMMMEHCyUw+r+uai5z73OfM9ybJfHLuM3OOOecQEREJhc/rAkREJHwoNEREJGQKDRERCZlCQ0REQqbQEBGRkEV5XUB5q1evnmvZsqXXZYiIhI2MjIy9zrn6pa2L+NBo2bIl6enpXpchIhI2zOyLM63T9JSIiIRMoSEiIiFTaIiISMgUGiIiEjKFhoiIhEyhISIiIVNoiIhIyBQaEjY++vIAL6bvQJfzF/FOxH+4T8LfZzmHeHh5Fu9k7QFg/a7D3HddB8zM48pEKh+FhlRYG746zCPLN7E8czc146P51dB27D1yitkfbKPIOX5/fUcFh8hFptCQCic79wiPvrWZNz79iuqxUdx9VSITL2tJ9bhonHNE+40Zq7ZSWOR4YFgnfD4Fh8jFotCQCmP73mM8tmIzr67bSVy0nzsGtmFy/9bUqhLz7z5mxj3XtMfnM556ZwtFzvGHGzorOEQuEoWGeG7H/uP87e3NvPTRTqL9xu39WzP18tbUrRZban8z41dXt8NvxhMrsykqgj/dpOAQuRgUGuKZrw6d4Im3s1mcvgPDGJt8CT8e2IYG1ePOua2Z8Yshifh8xuMrNlPoHH+5uQt+BYdIuVJoyEWXeySPJ1duYcHaL3HOMSKpOXcOakvjmvFl2o+Z8fPBifgMpr21maIix0PDuyo4RMqRQkMumn1HTzJj1Vbmrt5OfqHj5h5N+cmgBJrXqfKd9nvXVYn4zXh4+SaKnOOvw7sS5ddHkETKg0JDyt2h4/mkvLeFOR9s53h+ITd0a8pPr0ygVb2qF+w5fnJlAj6f8dCyLAodPDpCwSFSHhQaUm6O5OUz+/3tzHp/K0fyCri2S2PuviqBtg2ql8vz3TGwLX6f8ed/baTIOaaN7Ea0gkPkglJoyAV37GQBz63eTsqqrRw8ns/gDg25+6pEOjSpUe7P/cMr2uA34w//3EBRkePx0d0VHCIXkEJDLpi8/EKeT/2Cp97Zwr5jpxjQrj4/H5xIl2a1Lmodky9vjRn83xsbuHPBR/xtdA9iohQcIheCQkO+s5MFhSxau4PpK7PJPXKSfm3r8vPB7eh5SW3Parq9f2v8PuP3/8jkjgUfMX2MgkPkQlBoyHnLLyzixfQcnnh7M7sO5dG7ZR0eH92d5NZ1vS4NgAn9WuH3Gb99dT0/ej6DJ3/Qg9gov9dliYQ1hYaUWUFhEa98vJPH397Mjv0n6Na8Fn+5pQuXta1X4S4geFvflpgZ/7vkc344L4OnftCTuGgFh8j5Cul43cyGmlmWmWWb2T1n6DPCzDLNbL2ZLQhqX2pmB83s9RL93zOzdYHHLjNbUmJ9LzMrNLNbgtrGmdnmwGNc2YYq31VhkePVdTsZ8ugqfvn3T6kRF83s8Um88uPv0T+hfoULjNPGJl/CH2/szMqsPUydl0FefqHXJYmErXMeaZiZH5gODAZygDQze805lxnUJwG4F+jnnDtgZg2CdvEQUAWYGrxf51z/oO1fAl4t8Zx/AZYFtdUB7gOSAAdkBOo4EPpw5XwUFTmWrf+aR9/axKbdR2nXsDpP/6AnV3dsWGGDoqQxfVrgM7j3lc+YPDedmbcl6YhD5DyEMj3VG8h2zm0FMLNFwDAgM6jPZGD66Rdw51zu6RXOuRVmNuBMOzez6sAgYEJQ80+Al4BeQW1XA8udc/sD2y0HhgILQxiDnAfnHCs25PLI8k1kfnWY1vWr8vjo7ny/c+OwvDjgqN4t8PmMX7/0KZOeS2PWbb2Ij1FwiJRFKKHRFNgRtJwD9CnRJxHAzD4A/MDvnHNLQ6zhRmCFc+5wYB9NA22D+GZolFZH09J2aGZTgCkALVq0CLEMOc05x6rNe3lk+SY+2XGQFnWq8PDwrgzr1iTsP2U9Iqk5PjN++fdPmDgnjWfGJ1ElRqf2REIVym9LaX9SlrxJcxSQAAwAmgHvmVkn59zBEPY/GpgVtDwN+LVzrrDE1EcodRQ3OpcCpAAkJSXphtJl8OGWvTzy5ibSvzhA01rx/Pmmztzcs1lEfUDulp7N8PvgF4s/YcKzacwe34uqsQoOkVCE8puSAzQPWm4G7CqlT6pzLh/YZmZZFIdI2tl2bGZ1KZ7+ujGoOQlYFAiMesB/mVlB4DkGlKjjnRDqlxCkb9/Pw29uYvXWfTSsEcsDwzoyolfziH2L6o3dm+Ez4+4X1hUHx4ReVFNwiJxTKL8laUCCmbUCdgKjgDEl+iyh+IhhjpnVo3i6amsI+x4OvO6cyzvd4JxrdfprM5sTWL8kcCL8j2Z2+hNjQyg++S7fwSc7DvLw8k2s2rSHetVi+N/vd+DWPi0qxUniYd2a4jPjrhfWMX72Wp6d0IvqcdFelyVSoZ0zNJxzBWZ2J8XvZPIDs51z683sfiDdOfdaYN0QM8sECoFfOuf2QfFba4H2QDUzywEmOedOvytqFPDnUAp1zu03swf4z9HL/adPikvZrd91iEeXb+KtDbnUrhLNPde057a+l1S6+f3rujbBZ8ZPF33MuNlreW5ibwWHyFmYc5E95Z+UlOTS09O9LqPC2LT7CNPe2sQ/P/ua6nFRTOnfmvH9Wlb6F8qln3/FnQs+pnOzmjw3sTc1Kvn/h1RuZpbhnEsqbV3l+rOyEtu65yiPrdjMa5/sokq0n58Masvtl7WmZhW9OAIM7dSY6bcady74iLHPrGXuxN7UjNf/jUhJOtKIcDv2H+exFZt5+aMcYqP83Pa9S5h6eRvqVI3xurQK6a3M3fxofgaXNq7BvIl9FKpSKZ3tSEOhEaF2HTzB397O5sX0Hfh8xg/6XMKPBrShfvVYr0ur8N7euJsfzvuIxEbVeH5SH2pVUcBK5aLQqEShkXs4j+krs1m4dgcOx6heLbhjYFsa1YzzurSwsjIrl6nzMmhbvxrzb+9DbR2ZSSWi0KgEobH36EmefmcL81K/oKDIMbxnM+4c1JZmtat4XVrYenfTHqbMTadVvarMv70PdavpKE0qB4VGBIfGweOnSFm1lTkfbicvv5AbujflZ1cmcEndql6XFhHe37yXSc+l0bJuVeZP7kM9BYdUAgqNCAyNw3n5PPPeNma/v42jpwr4fpcm/OzKBNo2qOZ1aRHnw+y9THwujea1q7BgcrLOC0nEU2hEUGgcO1nAnA+3k7JqK4dO5HN1x4bcPTiR9o1qeF1aRFu9ZR8T56TRtHY8Cyb3oUF1nSOSyKXQiIDQOHGqkHmp23n63a3sP3aKQe0b8PPBiXRqWtPr0iqNNVv3MWFOGo1qxrFwcjINayg4JDIpNMI4NPLyC1m49kuefGcLe46cpH9CPe4enEiPFrXPvbFccGnb9zN+9loa1CgODr0rTSKRQiMMQ+NUQREvZuzgibez+epQHn1a1eEXQ9rRu1Udr0ur9DK+2M+42WnUqxbDwinJNK4Z73VJIheUQiOMQqOgsIiXP97J4ys2k3PgBD1a1OIXQ9rxvTZ1w+bWqpXBR18eYNwza6ldtTg4mtZScEjkUGiEQWgUFjn+8ckuHluxmW17j9G5aU1+PiSRAYn1FRYV1LodBxn7zBpqVYlm4eRkfSZGIsbZQiNybscWpoqKHG98+hVDp63irhfWERvlI2VsT167sx8D2zVQYFRg3ZrXYv7tfTh0PJ+RM1LZsf+41yWJlDuFhkecc7y5/muu/dv73LHgIxwwfUwP/vnT/gzp2EhhESa6NKvFgsnJHD1ZwKiUVL7cp+CQyKbQuMicc6zMymXY9A+YMi+DE6cKeHRkV5bddTnXdmmMz6ewCDedmtZk/u19OHaqgJEpq9m+95jXJYmUG53TuIg+zN7Lw8s3kfHFAZrWiudnVyZwU4+mRPmV3ZEgc9dhbp2VSmyUn4VTkmlVT5dykfCkE+Eeh0ba9v08/GYWqVv306hGHHcOasuIpObERCksIs3Grw8zZuYaonzGwinJtKmvy7pI+FFoeBQa63Yc5OE3s3hv817qVYvljoFtGN27BXHRfk/qkYsj6+sjjJmZis9nLJzch7YNqntdkkiZKDQucmh8vvMQjy7fxIqNudSpGsMPr2jN2OSWxMcoLCqLzbuPMHrmGgAWTu5DQkMFh4QPhcZFCo2sr4/w6PJNLF3/NTXioph6RRvGfa8l1WJ1K/bKKDv3KKNnplJU5FgwOZl2jRQcEh4UGuUcGlv2HGXaW5t5/dNdVI2JYtJlrZjUvxU14nR/6cpuy56jjE5JpaDIMf/2PlzaWFcjlopPoVFOofHFvmM8tmIzSz7eSWyUn/H9WjKlf2vdGlS+YdveY4xOSeVkQSHP396Hjk10ZWKp2BQaFzg0dh48wd9WbObvGTn4fcbY5Ev44YA2uqubnNH2vccYPTOVE/mFPD+pjy5pLxWaQuMChcbuw3lMX5nNorU7ABjduzk/HthW91WQkHy57zijZ6Zy9GQBz0/qQ+dmCg6pmBQa3zE09h49yVPvbOH51C8oLHIMT2rOnYPa6sqmUmY79h9nVEoqR/LymTepD12b1/K6JJFvUWicZ2gcOHaKGau28tyH2zlZUMhNPZrx00EJtKirq5nK+cs5UHzEcfB4PnMn9qa7bqglFYxCo4yhceJUIU+9k83sD7Zz7FQB13dtws+uTKC1Pt0rF8jOgycYnZLKgWOnmDOxNz0vUXBIxaFLo5eR32e8sm4nlyfWY9ldl/PYqO4KDLmgmtaK54WpydStFsO42WvJ+GK/1yWJhERHGmdw9GSBPpQn5e7rQ3mMmZnK7sN5zJnYm14tdTtf8Z6ONM6DAkMuhkY141g0JZmGNeMYN3sta7bu87okkbNSaIh4rEGN4uBoUiue8c+msXqLgkMqrpBCw8yGmlmWmWWb2T1n6DPCzDLNbL2ZLQhqX2pmB83s9RL93zOzdYHHLjNbEmgfZmafBtrTzeyyoG0Kg7Z57fyGLFLxNKgex8LJyTSvE8+EOWv5IHuv1yWJlOqc5zTMzA9sAgYDOUAaMNo5lxnUJwFYDAxyzh0wswbOudzAuiuBKsBU59z3z/AcLwGvOufmmlk14JhzzplZF2Cxc659oN9R51yZzkhXhPtpiIRq79GT/GDWGrbtPcascUn0T6jvdUlSCX3Xcxq9gWzn3Fbn3ClgETCsRJ/JwHTn3AGA04ER+HoFcOQsxVUHBgFLAv2Puv8kWVUgss/UiwSpVy2WBZOL7/o36bl03t20x+uSRL4hlNBoCuwIWs4JtAVLBBLN7AMzSzWzoWWo4UZghXPu8OkGM7vRzDYCbwATg/rGBaasUs3shjPt0MymBPql79mjXzoJL3WqxrBwcjJt61dj8tx0VmblnnsjkYsklNCwUtpK/vUfBSQAA4DRwCwzC/X6CKOBhd/YuXOvBKakbgAeCFrVInDINAaYZmZtStuhcy7FOZfknEuqX1+H9xJ+aleNYcHkPiQ2rMbUuRms2LDb65JEgNBCIwdoHrTcDNhVSp9XnXP5zrltQBbFIXJWZlaX4umvN0pb75xbBbQxs3qB5V2Bf7cC7wDdQ6hfJCzVqhLD/EnJtG9cnR8+n8HyTAWHeC+U0EgDEsyslZnFAKOAku9cWgIMBAi8wCcCW0PY93Dgdedc3ukGM2trZhb4ugcQA+wzs9pmFhv0HP2AzFL2KRIxalaJZt6kPnRoUpMfz89g2fqvvS5JKrlzhoZzrgC4E1gGbKD43Uzrzex+M7s+0G0ZxS/smcBK4JfOuX1Q/NZa4EXgSjPLMbOrg3Y/ihJTU8DNwOdmtg6YDowMnBi/FEg3s08Cz/Hn4HdwiUSqmvHRzJvUm05Na3LH/I/412dfeV2SVGK6jIhImDiSl8/4Z9NYt+Mgj4/qzrVdGntdkkQoXUZEJAJUj4vmuYm96dGiFj9d9DH/+KTkqUWR8qfQEAkj1WKjmDOh+FLqP1v0Ma+u2+l1SVLJKDREwkzV2CjmTOhF71Z1uPuFdbzycY7XJUklotAQCUNVYqJ4dnxvklvX5eeLP+HvGQoOuTgUGiJhKj7GzzPjetGvTT1++fdPWJy249wbiXxHCg2RMBYf42fWuCQua1uPX730KYvWful1SRLhFBoiYS4u2s/M25K4IrE+97z8GfPXfOF1SRLBFBoiESAu2s+MsT0Z2K4+//PK58xbvd3rkiRCKTREIkRctJ+nx/bkqksb8L+vrue5D7d7XZJEIIWGSASJjfLz5K09GdyhIfe9tp5nP9jmdUkSYRQaIhEmJsrH9DE9uLpjQ37/j0xmvRfKtUNFQqPQEIlAMVE+nhjTg//q3Ij/e2MDKau2eF2SRIgorwsQkfIR7ffx2KjumK3jj//cSGER/GhAqfctEwmZQkMkgkX7fTw2sht+M/6ydCNFznHHwLZelyVhTKEhEuGi/D4eGdEVn8FDy7IoLHL89Mpz3lhTpFQKDZFKIMrv4+ER3fD5jEeWb6LIOe66KtHrsiQMKTREKgm/z3jolq74zJj21maKihx3D04kcHdlkZAoNEQqEb/PePDmLvjNePztbAqd47+HtFNwSMgUGiKVjM9n/Ommzvh8xvSVWygsgl8PVXBIaBQaIpWQz2f84YZO+H3w9LtbKHKOe69pr+CQc1JoiFRSPp/xwLBO+MxIWbWVwiLHb669VMEhZ6XQEKnEzIzfX98RnxnPvL+NIuf47fc7KDjkjBQaIpWcmXHfdR3w+wLBUeT43fUdFRxSKoWGiGBm/ObaS/H7AlNVznH/9Z3w+RQc8k0KDREBioPj3mva4zPj6XeL31X1hxsUHPJNCg0R+Tcz49dD2+H3wfSVW3DO8ccbOys45N8UGiLyDWbGfw9p958PABY5/nxzF/wKDkGhISKlMDN+PqQdPl/xJUcKneOhW7oqOEShISJndtdVifis+CKHzsFfhys4KjuFhoic1U+vTMBn8Nc3N1FY5HhkRFei/LrpZ2Wl0BCRc7pzUAI+n/Hg0iyKnGPayG4KjkpKoSEiIfnxgLb4zfjTv4rvAPjYqO5EKzgqnZC+42Y21MyyzCzbzO45Q58RZpZpZuvNbEFQ+1IzO2hmr5fo/56ZrQs8dpnZkkD7MDP7NNCebmaXBW0zzsw2Bx7jzm/IInK+pl7Rht9ceyn//OxrfrLgY04VFHldklxk5zzSMDM/MB0YDOQAaWb2mnMuM6hPAnAv0M85d8DMGgTt4iGgCjA1eL/Ouf5B278EvBpYXAG85pxzZtYFWAy0N7M6wH1AEuCAjEAdB8o6aBE5f7f3b42Z8cDrmdyx4COmj+lBTJSOOCqLUL7TvYFs59xW59wpYBEwrESfycD00y/gzrnc0yuccyuAI2fauZlVBwYBSwL9jzrnXGB1VYoDAuBqYLlzbn/geZYDQ0OoX0QusEmXteJ313VgeeZufjw/g5MFhV6XJBdJKKHRFNgRtJwTaAuWCCSa2QdmlmpmZXkxvxFY4Zw7fLrBzG40s43AG8DEMtQhIhfJ+H6teGBYR97akMvUeRnk5Ss4KoNQQqO0N2W7EstRQAIwABgNzDKzWiHWMBpY+I2dO/eKc649cAPwQBnqKO5oNiVwPiR9z549IZYhImU1tm9L/nRTZ97dtIcJz6Zx7GSB1yVJOQslNHKA5kHLzYBdpfR51TmX75zbBmRRHCJnZWZ1KZ7+eqO09c65VUAbM6sXYh2nt0txziU555Lq169/rjJE5DsY3bsFj4zoytrt+7lt9loO5+V7XZKUo1BCIw1IMLNWZhYDjAJeK9FnCTAQIPACnwhsDWHfw4HXnXN5pxvMrK0FLuRvZj2AGGAfsAwYYma1zaw2MCTQJiIeu7F7M54Y3Z1Pcw5y68w1HDh2yuuSpJycMzSccwXAnRS/QG8AFjvn1pvZ/WZ2faDbMmCfmWUCK4FfOuf2QfFba4EXgSvNLMfMrg7a/ShKTE0BNwOfm9k6it+1NdIV20/xVFVa4HF/oE1EKoBrOjcmZWwSWbuPMCollT1HTnpdkpQD+88blSJTUlKSS09P97oMkUrjg+y93P5cOo1rxjF/ch8a14z3uiQpIzPLcM4llbZOb64WkQuqX9t6zJ3Um9wjJxkxYzU79h/3uiS5gBQaInLB9WpZh/m39+HwiQKGP72arXuOel2SXCAKDREpF12b12LRlGTyC4sYMSOVrK/P+BlfCSMKDREpN5c2rsELU/vi98HIlNV8lnPI65LkO1JoiEi5atugGoun9qVqTBRjZqaS8YXe9BjOFBoiUu4uqVuVF3/Yl3rVYxn7zFo+3LLX65LkPCk0ROSiaFIrnhemJtOsdjwTnk3jnazcc28kFY5CQ0QumgbV41g0pS9tG1Rj8tx0ln7+tdclSRkpNETkoqpTNYYFk5Pp1LQmdyz4iFfX7fS6JCkDhYaIXHQ146OZN6kPvVrW5q4X1rE4bce5N5IKQaEhIp6oFhvFs+N70z+hPr966VOe+3C71yVJCBQaIuKZ+Bg/M2/ryeAODbnvtfU8/e4Wr0uSc1BoiIinYqP8PHlrD67r2oQ//2sjjy7fRKRfSDWcRXldgIhItN/HtJHdiI3y8diKzeTlF3LPNe0J3FpHKhCFhohUCH6f8eDNXYiP9jNj1VZO5Bfyu+s64vMpOCoShYaIVBg+n3H/sI7Ex/hJWbWVvPxC/nRTF/wKjgpDoSEiFYqZce817YmP9gemqop4eERXov06BVsRKDREpMIxM+4enEhctJ+/LN1IXn4hfxvTndgov9elVXqKbhGpsH40oA2/u64Db2buZsrcDE6cKvS6pEpPoSEiFdr4fq34y82dWbV5DxPmrOXYyQKvS6rUFBoiUuGN7NWCaSO7kbb9AGOfWcOhE/lel1RpKTREJCwM69aU6WO689nOQ9w6K5X9x055XVKlpNAQkbAxtFNjUsYmsXn3UUalrCb3SJ7XJVU6Cg0RCSsD2zfg2fG9yDlwgpEzUtl18ITXJVUqCg0RCTvfa1uPuRN7s/fISUbMWM2X+457XVKlodAQkbCU1LIO8yf34ejJAkbMWE127lGvS6oUFBoiEra6NKvFoinJFBQVMSplNRu/Pux1SRFPoSEiYa19oxq8MLUvUT4fo1JS+TTnoNclRTSFhoiEvTb1q7F4al+qxUZx68w1pG/f73VJEUuhISIRoUXdKiye2pf61WMZ+8xaPsze63VJEUmhISIRo0mteBZNTaZFnSqMn5PGyo25XpcUcRQaIhJRGlSPY9GUZBIbVmPKvHSWfv6V1yVFFIWGiESc2lVjmH97Mp2b1uSOBR/z6rqdXpcUMUIKDTMbamZZZpZtZvecoc8IM8s0s/VmtiCofamZHTSz10v0f8/M1gUeu8xsSaD9VjP7NPD40My6Bm2z3cw+C2yTfn5DFpHKoGZ8NPMm9aFXy9rc9cI6Fq390uuSIsI5b8JkZn5gOjAYyAHSzOw151xmUJ8E4F6gn3PugJk1CNrFQ0AVYGrwfp1z/YO2fwl4NbC4DbgisJ9rgBSgT9CmA51zOsMlIudUNTaKORN6M3VeBve8/Bl5+YWM79fK67LCWihHGr2BbOfcVufcKWARMKxEn8nAdOfcAQDn3L/PPjnnVgBHzrRzM6sODAKWBPp/eHo/QCrQLMSxiIh8S1y0n5TbejKkQ0N+949Mnnpni9clhbVQQqMpsCNoOSfQFiwRSDSzD8ws1cyGlqGGG4EVzrnSPso5CfhX0LID3jSzDDObcqYdmtkUM0s3s/Q9e/aUoRQRiUSxUX6m39qD67o24S9LN/LI8k0457wuKyyFco9wK6Wt5P92FJAADKD4yOA9M+vknAvlo5mjgVnfelKzgRSHxmVBzf2cc7sC01/LzWyjc27Vt4pzLoXiaS2SkpL0kyEiRPt9TBvZjfhoH4+v2ExefiH3XtMes9Je4uRMQgmNHKB50HIzYFcpfVKdc/nANjPLojhE0s62YzOrS/H0140l2rtQHCTXOOf2nW53zu0K/JtrZq8Etv1WaIiIlMbvM/58Uxfio/2krNrKiVOF/P76jvh8Co5QhTI9lQYkmFkrM4sBRgGvleizBBgIYGb1KJ6u2hrCvocDrzvn/n0nFTNrAbwMjHXObQpqrxo4/4GZVQWGAJ+H8BwiIv/m8xm/u74jUy9vzbzUL/jVS59SWKQJiVCd80jDOVdgZncCywA/MNs5t97M7gfSnXOvBdYNMbNMoBD45ekjBDN7D2gPVDOzHGCSc25ZYPejgD+XeMrfAnWBJwOHjQXOuSSgIfBKoC0KWOCcW/odxi4ilZSZcc817YmP8TPtreKpqkdHdiPar4+unYtF+smgpKQkl56uj3SISOlmvLuFP/1rI4M7NOSJMd2JjfJ7XZLnzCwj8Mf6tyhWRaRSm3pFG35/fUeWZ+5m8twMTpwq9LqkCk2hISKV3rjvteTBm7vw3uY9jH92LUdPFnhdUoWl0BARAUb0as60kd1I/+IAY59Zw6ET+V6XVCEpNEREAoZ1a8r0MT34fOchxsxMZf+xU16XVOEoNEREggzt1IiZtyWRnXuUUSmryT2cd+6NKhGFhohICQPaNeDZCb3IOXCCkSmp7Dp4wuuSKgyFhohIKb7Xph7zJvVm75GTDH96NV/uO+51SRWCQkNE5Ax6XlKHBZOTOXaqgOEzPiQ796jXJXlOoSEichadm9XkhSl9KSyCkTNWs+Gr0i7IXXkoNEREzqFdo+q8MDWZaL+PUSmpfLIjlAt4RyaFhohICNrUr8aLP+xLjfgobp21hrTt+70uyRMKDRGREDWvU4XFU/vSoHostz2zlg+yK9+dpxUaIiJl0LhmPC9M7UuLOlWYMCeNtzfu9rqki0qhISJSRvWrx7JoSjLtGlZn6rwM/vXZV16XdNEoNEREzkPtqjHMn9yHLs1qcefCj1ny8U6vS7ooFBoiIuepRtizXCQAAAodSURBVFw0cyf2pnfLOty9eB0L137pdUnlTqEhIvIdVI2N4tkJvbgisT73vvwZs9/f5nVJ5UqhISLyHcVF+5kxtidXd2zI/a9n8uQ72V6XVG4UGiIiF0BslJ/pY3owrFsTHlyaxSNvZhGJt9OO8roAEZFIEeX38ciIbsRF+Xn87WyOnyrkf669FDPzurQLRqEhInIB+X3Gn27qTHyMn1nvb+NEfiEPDOuEzxcZwaHQEBG5wHw+477rOhAX7efpd7eQl1/Eg7d0wR8BwaHQEBEpB2bGr4e2Iz7az6NvbeJkQSGPjuxGtD+8TyUrNEREyomZ8bOrEoiP8fHHf24kL7+IJ8Z0Jy7a73Vp5y28I09EJAxMubwNDwzryFsbdjN5bjonThV6XdJ5U2iIiFwEY/u25MFbuvBB9l7GPbuWoycLvC7pvCg0REQukhFJzZk2qjsZXxzgB7PWcOh4vtcllZlCQ0TkIrq+axOevLUHmbsOM3pmKvuOnvS6pDJRaIiIXGRXd2zEzHFJbNlzlFEpqeQezvO6pJApNEREPHBFYn3mTOjNzoMnGDFjNTsPnvC6pJAoNEREPNK3TV3mTerDvmOnGPH0ar7Yd8zrks5JoSEi4qGel9Rm4eRkjp8qYPjTq8nOPeJ1SWcVUmiY2VAzyzKzbDO75wx9RphZppmtN7MFQe1Lzeygmb1eov97ZrYu8NhlZksC7bea2aeBx4dm1rUsdYiIhJtOTWuyaEpfihyMnJFK5q7DXpd0RucMDTPzA9OBa4AOwGgz61CiTwJwL9DPOdcRuCto9UPA2JL7dc71d851c851A1YDLwdWbQOucM51AR4AUkKtQ0QkXLVrVJ3FU5OJifIxemYq63Yc9LqkUoVypNEbyHbObXXOnQIWAcNK9JkMTHfOHQBwzuWeXuGcWwGc8XjLzKoDg4Algf4fnt4PkAo0K0MdIiJhq3X9aiye2pca8VH8YNYa1m7b73VJ3xJKaDQFdgQt5wTagiUCiWb2gZmlmtnQMtRwI7DCOVfa8dgk4F9lqAMAM5tiZulmlr5nz54ylCIi4q3mdarw4tTv0aBGLONmr+X9zXu9LukbQgmN0q7lW/J2VFFAAjAAGA3MMrNaIdYwGlj4rSc1G0hxaPy6DHUUNzqX4pxLcs4l1a9fP8QyREQqhkY143hhSl8uqVuFic+l8fbG3V6X9G+hhEYO0DxouRmwq5Q+rzrn8p1z24AsikPkrMysLsXTTm+UaO8CzAKGOef2laEOEZGIUL96LIumJNO+UXWmzM3gn5995XVJQGihkQYkmFkrM4sBRgGvleizBBgIYGb1KJ6u2hrCvocDrzvn/v1xSDNrQfFJ8bHOuU1lrENEJGLUqhLD87f3oVvzWty54CNe/ijH65LOHRrOuQLgTmAZsAFY7Jxbb2b3m9n1gW7LgH1mlgmsBH55+gjBzN4DXgSuNLMcM7s6aPej+PbU1G+BusCTgbfjpp+tjvMatYhImKgRF83cSb1Jbl2XX7z4CQvWfOlpPeZcqacFIkZSUpJLT0/3ugwRke8kL7+QHz2fwcqsPfz2+x2YeFmrcnsuM8twziWVtk6fCBcRCQNx0X5mjE1iaMdG3P96JtNXZntSh0JDRCRMxET5eGJMd27o1oSHlmXx12VZXOzZIt0jXEQkjET5fTw8ohtx0X6eWJnNifxCfnPtpZiV9qmEcnj+i/IsIiJywfh9xp9u6kxctJ9n3t9GXn4hDwzrhM9X/sGh0BARCUNmxn3XdSA+xs9T72zhRH4hD97chSh/+Z51UGiIiIQpM+NXV7ejSrSfh5dv4mR+EdNGdSO6HINDoSEiEsbMjJ9cmUB8jJ//e2MDJwsKeWJMD+Ki/eXyfHr3lIhIBLi9f2seuKETb23IZfLcdE6cKiyX51FoiIhEiLHJl/DQLV34IHsv42av5fipggv+HJqeEhGJIMOTmhMX7ee9zXuIi7rwU1QKDRGRCHNd1yZc17VJuexb01MiIhIyhYaIiIRMoSEiIiFTaIiISMgUGiIiEjKFhoiIhEyhISIiIVNoiIhIyCL+HuFmtgf44jw3rwfsvYDleClSxhIp4wCNpSKKlHHAdxvLJc65+qWtiPjQ+C7MLP1MN1cPN5EylkgZB2gsFVGkjAPKbyyanhIRkZApNEREJGQKjbNL8bqACyhSxhIp4wCNpSKKlHFAOY1F5zRERCRkOtIQEZGQKTRERCRkCo0AM5ttZrlm9nlQWx0zW25mmwP/1vayxlCYWXMzW2lmG8xsvZn9LNAejmOJM7O1ZvZJYCy/D7S3MrM1gbG8YGYxXtcaCjPzm9nHZvZ6YDlcx7HdzD4zs3Vmlh5oC7ufLwAzq2VmfzezjYHfmb7hOBYzaxf4fpx+HDazu8pjLAqN/5gDDC3Rdg+wwjmXAKwILFd0BcAvnHOXAsnAHWbWgfAcy0lgkHOuK9ANGGpmycBfgEcDYzkATPKwxrL4GbAhaDlcxwEw0DnXLehzAOH48wXwGLDUOdce6Erx9yfsxuKcywp8P7oBPYHjwCuUx1icc3oEHkBL4POg5SygceDrxkCW1zWex5heBQaH+1iAKsBHQB+KP+UaFWjvCyzzur4Q6m8W+KUdBLwOWDiOI1DrdqBeibaw+/kCagDbCLwhKJzHUqL+IcAH5TUWHWmcXUPn3FcAgX8beFxPmZhZS6A7sIYwHUtgSmcdkAssB7YAB51zBYEuOUBTr+org2nAr4CiwHJdwnMcAA5408wyzGxKoC0cf75aA3uAZwPThrPMrCrhOZZgo4CFga8v+FgUGhHKzKoBLwF3OecOe13P+XLOFbriQ+5mQG/g0tK6XdyqysbMvg/kOucygptL6VqhxxGkn3OuB3ANxdOfl3td0HmKAnoATznnugPHCIOpqLMJnBe7HnixvJ5DoXF2u82sMUDg31yP6wmJmUVTHBjznXMvB5rDciynOecOAu9QfJ6mlplFBVY1A3Z5VVeI+gHXm9l2YBHFU1TTCL9xAOCc2xX4N5fiefPehOfPVw6Q45xbE1j+O8UhEo5jOe0a4CPn3O7A8gUfi0Lj7F4DxgW+Hkfx+YEKzcwMeAbY4Jx7JGhVOI6lvpnVCnwdD1xF8YnKlcAtgW4VfizOuXudc82ccy0pnjp42zl3K2E2DgAzq2pm1U9/TfH8+eeE4c+Xc+5rYIeZtQs0XQlkEoZjCTKa/0xNQTmMRZ8IDzCzhcAAii8nvBu4D1gCLAZaAF8Cw51z+72qMRRmdhnwHvAZ/5k//38Un9cIt7F0AZ4D/BT/gbPYOXe/mbWm+C/2OsDHwA+ccye9qzR0ZjYA+G/n3PfDcRyBml8JLEYBC5xzfzCzuoTZzxeAmXUDZgExwFZgAoGfNcJvLFWAHUBr59yhQNsF/74oNEREJGSanhIRkZApNEREJGQKDRERCZlCQ0REQqbQEBGRkCk0REQkZAoNEREJ2f8Hml/ex06Rih8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# # plot CV误差曲线\n",
    "test_means = grid_search_min_child_samples_v1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search_min_child_samples_v1.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid_search_v1.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid_search_v1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "x_axis = min_child_samples_s\n",
    "\n",
    "plt.plot(x_axis, -test_means)\n",
    "# #plt.errorbar(x_axis, -test_scores, yerr=test_stds ,label = ' Test')\n",
    "# #plt.errorbar(x_axis, -train_scores, yerr=train_stds,label =  +' Train')\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "### 继续增大 min_child_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done   8 out of  15 | elapsed: 280.0min remaining: 245.0min\n",
      "[Parallel(n_jobs=6)]: Done  12 out of  15 | elapsed: 353.7min remaining: 88.4min\n",
      "[Parallel(n_jobs=6)]: Done  15 out of  15 | elapsed: 371.7min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      class_weight=None, colsample_bytree=0.7,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=64,\n",
       "                                      max_depth=7, min_child_samples=20,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=9431,\n",
       "                                      n_jobs=6, num_leaves=30,\n",
       "                                      objective='regression', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.7,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='warn', n_jobs=6,\n",
       "             param_grid={'min_child_samples': range(70, 160, 20)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':30,\n",
    "          'max_bin': 64, # 稀疏离散特征一共有167维，先不设置很大，提高训练速度\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "#           'early_stopping_round':10, # 添加 early_stopping_round，减少迭代次数\n",
    "#           'metric':'auc'\n",
    "         }\n",
    "lg_v1 = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "min_child_samples_s = range(70,160,20) \n",
    "tuned_parameters = dict( min_child_samples = min_child_samples_s)\n",
    "\n",
    "grid_search_min_child_samples_v2 = GridSearchCV(lg_v1, n_jobs=6,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search_min_child_samples_v2.fit(train_X , train_Y)\n",
    "# grid_search_v1.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(model_path + 'grid_search_min_child_samples_v2.pkl','wb') as fw:\n",
    "        pk.dump(grid_search_min_child_samples_v2,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump grid_search_min_child_samples_v2 error,',e)\n",
    "try:\n",
    "    # 序列化到磁盘中\n",
    "    with open(model_path + 'lg_v2.pkl','wb') as fw:\n",
    "        pk.dump(lg_v1,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg_v1 error,',e)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f73b6b3c860>]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD4CAYAAAAQP7oXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU9b3G8c+XhC0krIGwhLAmrIJgBOqCSBXXqriC1lq1YrVoW6ttrb2ut61e29vWintta69ItSpE3IsiuLMLQQIhbGEJELYECNm+948ZYIxBJprkZHner9e8kjnnzJlnZpJ5cn7nzIm5OyIiItFoEnQAERGpP1QaIiISNZWGiIhETaUhIiJRU2mIiEjUYoMOUNMSExO9Z8+eQccQEak3FixYsN3dO1Y2r8GXRs+ePZk/f37QMURE6g0zW3ekeRqeEhGRqKk0REQkaioNERGJmkpDRESiptIQEZGoqTRERCRqUZWGmZ1pZllmlm1mvzzCMpea2XIzyzSzqRHT3zCzXWY2s8Lyc81scfiyycymV5h/vJmVmdnFEdOuMrNV4ctVVXuoIiLyTR31cxpmFgNMAU4HcoF5Zpbh7ssjlkkFbgdOdPedZtYpYhUPAnHA9ZHrdfeTI27/IjCjwn0+ALwZMa09cBeQDjiwIJxjZ/QPV0Sk4fskJ5/MTXu4+sSemFm1rjuaLY0RQLa757h7MTANOL/CMtcBUw6+gbv71oMz3H0WUHCklZtZAjAWiNzSuAl4EdgaMe0M4G133xG+n7eBM6PILyLSaOQXHuDmaYt45qO17C8pq/b1R1Ma3YANEddzw9MipQFpZvaBmX1sZlV5Mx8PzHL3PQBm1i087bGvkUNEpNEqL3d+9sISdu4t4eHLhxPXrPpP+hHNGivbtqn47/5igVRgDJAMzDWzwe6+K4r1TwSeirj+J+AX7l5WYbMqmhyhBc0mAZMAUlJSooggIlL/PfV+DrOztnHv+YMY3K1NjdxHNKWRC3SPuJ4MbKpkmY/dvQRYY2ZZhEpk3let2Mw6EBr+Gh8xOR2YFi6MROBsMysN38eYCjlmV7Zed38CeAIgPT1d/89WRBq8het38j9vZHHmoM5cOapHjd1PNMNT84BUM+tlZs2ACUBGhWWmA6cCmFkioeGqnCjWfQkw092LDk5w917u3tPdewL/Bm509+mEdoqPM7N2ZtYOGEfEjnIRkcZq974Sbpq6iM5tWvDAxUOqfed3pKNuabh7qZlNJvQGHQM87e6ZZnYvMN/dMzj8hr4cKANuc/d8CB1aC/QH4s0sF7jW3Q++2U8A7o8mqLvvMLP7OLz1cq+774j6kYqINEDuzs9fXELeniJe+OG3aNOyaY3en7k37NGb9PR016nRRaSheuajtdw5I5M7zh7AdaN7V8s6zWyBu6dXNk+fCBcRqaeWbdzNf8/8nFP7deTak3rVyn2qNERE6qHCA6VMnrqQdq2a8odLj6VJk5rbjxGpwf/nPhGRhsbd+dVLS1m/Yx/PXTeK9q2a1dp9a0tDRKSeeX7+BjKWbOKnp6UxsneHWr1vlYaISD2yMq+AuzIyObFvB248tW+t379KQ0SknthfXMaPnl1IfPNY/njZscTU0n6MSNqnISJST9ydkUn2tkL+ec1IOiW0CCSDtjREROqB6Ys28q/5G7hxTB9OSk0MLIdKQ0SkjsvZVsgdLy8lvUc7fnpaWqBZVBoiInVYUUkZk6cuomlsEx6aOIzYmGDftrVPQ0SkDvvta5+zfPMenvpeOl3btgw6jrY0RETqqteXbuaZj9bxg5N6cdrApKDjACoNEZE6acOOffz8xc8YmtyGn5/ZP+g4h6g0RETqmOLSciY/twiAhy8fTrPYuvNWrX0aIiJ1zO/fymLJhl08csVwurePCzrOF9Sd+hIREd5ZkccTc3L47qgUzj6mS9BxvkSlISJSR2zevZ+fPb+E/p0T+PU5A4OOUymVhohIHVBaVs6Pn1vMgdJyplwxnBZNY4KOVCnt0xARqQMemrWKT9fu4I+XDaVPx/ig4xyRtjRERAL2QfZ2/vJuNpccl8z4YclBx/lKKg0RkQBtKzjAj6ctpk/HeO45f1DQcY5Kw1MiIgEpL3d++q/FFBSV8OwPRhLXrO6/Jdf9hCIiDdSj763m/ezt/O7CY+jXOSHoOFHR8JSISAA+XbODP7yVxXeGdmXC8d2DjhM1lYaISC3bubeYH09bRPf2cfx2/GDMav/ftn5dGp4SEalF7s6tLywhv7CYl248gYQWTYOOVCXa0hARqUV/fX8Ns1Zs5Vdn92dwtzZBx6kylYaISC1ZvGEXD7yxgnEDk7jqhJ5Bx/laVBoiIrVg9/4SbnpuIZ0SWvDgxUPr1X6MSNqnISJSw9yd21/6jE27inj++m/RJq5+7ceIpC0NEZEa9n+frOe1pVu47Yx+HNejXdBxvhGVhohIDcrctJv7Zi7nlLSOTDq5d9BxvjGVhohIDdl7oJSbpi6iXVxT/vfSoTRpUj/3Y0TSPg0RkRrg7vx6+jLW5u9l6nWj6BDfPOhI1UJbGiIiNeDfC3J5edFGfvztNEb17hB0nGqj0hARqWar8gq4c0Ym3+rdgclj+wYdp1pFVRpmdqaZZZlZtpn98gjLXGpmy80s08ymRkx/w8x2mdnMCsvPNbPF4csmM5senn6+mX0Wnj7fzE6KuE1ZxG0yvt5DFhGpOfuLy5g8dRFxzWL484RjiWkA+zEiHXWfhpnFAFOA04FcYJ6ZZbj78ohlUoHbgRPdfaeZdYpYxYNAHHB95Hrd/eSI278IzAhfnQVkuLub2RDgeaB/eN5+dz+2io9RRKTW3Dszk6y8Av5xzQg6tW4RdJxqF82Wxggg291z3L0YmAacX2GZ64Ap7r4TwN23Hpzh7rOAgiOt3MwSgLHA9PDyhe7u4dmtAD/SbUVE6pKMJZt47tMN3DCmD6ekdQw6To2IpjS6ARsirueGp0VKA9LM7AMz+9jMzqxChvHALHffc3CCmY03sxXAq8A1Ecu2CA9ZfWxmF1ThPkREatTa7Xv51UtLOa5HO245PS3oODUmmkNuKxuQq/jXfyyQCowBkoG5ZjbY3XdFsf6JwFNfWLn7y8DLZjYauA84LTwrxd03mVlv4B0zW+ruq78U2GwSMAkgJSUliggiIl/fgdIyJj+3kJgmxkMTh9E0puEeYxTNI8sFIv+tVDKwqZJlZrh7ibuvAbIIlchXMrMOhIa/Xq1svrvPAfqYWWL4+qbw1xxgNjDsCLd7wt3T3T29Y8eGuYkoInXH715bwbKNe/j9JUPp1rZl0HFqVDSlMQ9INbNeZtYMmABUPHJpOnAqQPgNPg3IiWLdlwAz3b3o4AQz62vh0z+a2XCgGZBvZu3MrHnEfZwILK9knSIitebNzC38/cO1XHNiL04fmBR0nBp31OEpdy81s8nAm0AM8LS7Z5rZvcB8d88IzxtnZsuBMuA2d8+H0KG1hI5+ijezXOBad38zvPoJwP0V7vIi4HtmVgLsBy4LH0k1AHjczMoJld39kUdwiYjUttyd+7jthSUMSW7DL8/qf/QbNAB2+EClhik9Pd3nz58fdAwRaWBKysq59PGPWJVXyKs3n0SPDq2CjlRtzGyBu6dXNk/nnhIR+Rr+8NZKFq3fxcOXD2tQhXE0DXcXv4hIDZmdtZXH3lvN5SNTOHdI16Dj1CqVhohIFeTtKeKW55fQv3MCd547MOg4tU6lISISpbJy5+bnFrG/uIyHLx9Oi6YxQUeqddqnISISpYdmreKTNTv4wyVD6dspPug4gdCWhohIFD5cvZ2H3lnFRcOTuei45KDjBEalISJyFNsLD/CTaYvpndiKe88fFHScQGl4SkTkK5SXO7c8v4Rd+0v4xzUjaNW8cb9taktDROQrPD4nhzkrt3HXdwYyoEvroOMETqUhInIEC9bt4PdvZXHOkC5cPkJnzAaVhohIpXbtK+amqYvo1rYlv7vwGMLnUW30GvfgnIhIJdydW1/4jG2FB3jxhhNo3aJp0JHqDG1piIhU8LcP1vKfz/O4/awBDEluG3ScOkWlISIS4bPcXfzu9c85bUASV5/YM+g4dY5KQ0QkbE9RCZOnLqJjfHN+f8kQ7ceohPZpiIgQ2o9x+0tL2bhrP/+aNIq2cc2CjlQnaUtDRAR47tMNvPrZZn42Lo30nu2DjlNnqTREpNH7fPMe7nklk5NTE/nh6D5Bx6nTVBoi0qjtPVDK5KkLad2yKX+87FiaNNF+jK+ifRoi0qjdOSOTnO17efYHI0mMbx50nDpPWxoi0mi9uCCXFxfmcvPYVE7okxh0nHpBpSEijVL21kL+a8YyRvVuz83fTg06Tr2h0hCRRqeopIzJUxfSomkMf54wjBjtx4ia9mmISKNz38zlrNhSwN+uPp6k1i2CjlOvaEtDRBqVmZ9t4tlP1nP9Kb05tV+noOPUOyoNEWk01ufv4/YXlzIspS23jusXdJx6SaUhIo1CcWk5k59biBn8ZeIwmsbo7e/r0D4NEWkUHnhjBZ/l7ubxK48juV1c0HHqLVWtiDR4by/P46/vr+H7J/TkjEGdg45Tr6k0RKRB27hrP7e+sITB3Vpz+9n9g45T76k0RKTBKikr5+bnFlFW7jw8cTjNY2OCjlTvaZ+GiDRYf3x7JQvW7eShicPomdgq6DgNgrY0RKRBmrNyG4/MXs3EEd05b2jXoOM0GCoNEWlwtu4p4pbnF9MvKYE7zx0UdJwGRcNTItKglJU7P/nXYvYeKOO564bRspn2Y1QnlYaINChT3s3mw9X5PHjxEFKTEoKO0+BENTxlZmeaWZaZZZvZL4+wzKVmttzMMs1sasT0N8xsl5nNrLD8XDNbHL5sMrPp4ennm9ln4enzzeykiNtcZWarwpervt5DFpGG6uOcfP70n5WMH9aNi49LDjpOg3TULQ0ziwGmAKcDucA8M8tw9+URy6QCtwMnuvtOM4s8C9iDQBxwfeR63f3kiNu/CMwIX50FZLi7m9kQ4Hmgv5m1B+4C0gEHFoRz7KzqgxaRhie/8AA/nraInh1acd8FgzHT6c5rQjRbGiOAbHfPcfdiYBpwfoVlrgOmHHwDd/etB2e4+yyg4EgrN7MEYCwwPbx8obt7eHYrQgUBcAbwtrvvCN/P28CZUeQXkQauvNz52QtL2LmvhL9cPoz45hp5rynRlEY3YEPE9dzwtEhpQJqZfWBmH5tZVd7MxwOz3H3PwQlmNt7MVgCvAtdUIYeINEJPzs1hdtY2/uvcgQzq2iboOA1aNKVR2TaeV7geC6QCY4CJwFNm1jbKDBOB576wcveX3b0/cAFwXxVyhBY0mxTeHzJ/27ZtUcYQkfpo4fqdPPhmFmcf05nvjkwJOk6DF01p5ALdI64nA5sqWWaGu5e4+xogi1CJfCUz60Bo+OvVyua7+xygj5klRpnj4O2ecPd0d0/v2LHj0WKISD21e18JN01dRJe2LfjdhUO0H6MWRFMa84BUM+tlZs2ACUBGhWWmA6cChN/g04CcKNZ9CTDT3YsOTjCzvhZ+5c1sONAMyAfeBMaZWTszaweMC08TkUbI3fn5i0vYWlDEwxOH06Zl06AjNQpH3Vvk7qVmNpnQG3QM8LS7Z5rZvcB8d8/g8Bv6cqAMuM3d8yF0aC3QH4g3s1zgWnc/+GY/Abi/wl1eBHzPzEqA/cBl4R3jO8zsPkIlBnCvu+/4+g9dROqzZz5ax5uZefz6nAEM7R7taLh8U3b4QKWGKT093efPnx90DBGpRss27ubCRz7k5NREnroqXcNS1czMFrh7emXzdO4pEalXCg+UMnnqQjrEN+P3lwxVYdQyHcwsIvWGu/Orl5ayYed+pk0aRbtWzYKO1OhoS0NE6o3n528gY8kmbjk9jeN7tg86TqOk0hCReiFrSwF3ZWRycmoiN5zSJ+g4jZZKQ0TqvH3Fof0Y8c2b8r+XHkuTJtqPERTt0xCROqW83Nmwcx9ZWwpYmVdAVl4hS3N3sW7HPv7v2pF0TGgedMRGTaUhIoFwdzbvLiIrr4BVeQVkbSlkZV4Bq7YWUFRSfmi55HYt6ZeUwE9OS+PEvokBJhZQaYhILdheeICVWwrIyitgZV6oHFZuKaDgQOmhZZJaNyctKYErRvagX1ICaZ0TSO0UTyudsbZO0ashItVm9/6S0FZDuBRCWxGF5O8tPrRMu7impCUlMH54N1KTEkIFkRRP2zgdPlsfqDREpMr2FZeyKq/w8NBSXiErtxSwZc+h08gR3zyW1KR4Th+YRFpSAv06J5CaFE/H+Ob6QF49ptIQkSM6UFpGzra9oR3S4R3TK/MK2bBzHwfPQNQ8tgmpSfGc0KcDaZ0TDg0tdW3TQuXQAKk0RITSsnLW5u87PLQULom1+fsoKw+1Q2wTo3fHVgxJbsMlxyWHhpY6J5DSPo4YHQLbaKg0RBqR8nJn4679ZB3a3xAaWlq9tZDistARS2bQo30caUkJnH1MF9KSEkhLSqBXYiuaxeqjXY2dSkOkAXJ38vYcCA8nHR5aWrW1kH3FZYeW69a2JWlJ8YxOTTy036FPx3haNosJML3UZSoNkXpux95israEPt9w6ANxWwrYU3T4cNbE+Ob06xzPZcd3P7TlkJoUT+sW+sdFUjUqDZF6oqCo5NBnHA6XRCHbCw8cWqZ1i1j6dU7gO0O70q9zwqGCaK+zwUo1UWmI1DH7i8vI3lp4eGgp/JmHTbsPH84a1yyG1KQETu3X8VA59OucQKcEHc4qNUulIRKQ4tJy1mzfe6gUDpbEuh2HD2dtFtOEPp3iGdGrPWmdE0jrFCqHbm1b6qR9EgiVhkgtcnfmrNrOY7NXM2/tDkrDh7PGNDF6dohjYNfWXDCs26HPOvRoH0dsjI5YkrpDpSFSC8rLnbeW5/HI7Gw+y91N59Yt+MHJvRnQJTS01LtjK5rH6oglqftUGiI1qLSsnFeXbmbKu9mszCskpX0cv7vwGC4c3k0lIfWSSkOkBhSXlvPSwlwefW816/L3kdopnj9ddiznDumi4Sap11QaItVof3EZ0+at54k5OWzeXcQx3drw2HePY9zAJO24lgZBpSFSDQqKSvjnx+v469w15O8tZkTP9tx/0RBGpybqEFhpUFQaIt/Azr3F/O2DNfz9w7XsKSpldFpHJp/alxG92gcdTaRGqDREvoate4p4cm4Oz36ynn3FZYwbmMTksX0Zktw26GgiNUqlIVIFG3bs4/E5q3l+fi6lZeV8Z2hXbhzTl36dE4KOJlIrVBoiUVi9rZBHZ69m+qKNmMFFw5P54Sl96JnYKuhoIrVKpSHyFZZv2sOU2dm8tnQzzWOb8N1RPZg0ujdd27YMOppIIFQaIpVYuH4nU97JZtaKrcQ3j+WHp/Th2pN6kRjfPOhoIoFSaYiEuTsfrc7n4Xez+XB1Pm3jmnLL6Wlc9a2etInT/50QAZWGCO7OOyu28vC72Sxav4uOCc254+wBXD4yhVbN9SsiEkm/EdJolZU7ry/bzJR3V/P55j10a9uS+y4YzCXHJdOiqc4LJVIZlYY0OiVl5cxYvIlHZmeTs20vvRNb8eDFQ7hgWDea6rxQIl9JpSGNRlFJGS8syOXx91aTu3M/A7q05uHLh3HW4C7E6LxQIlFRaUiDt/dAKVM/Wc+Tc3PYWnCAYSltuee8QYzt30nnhRKpoqhKw8zOBP4MxABPufv9lSxzKXA34MASd788PP0NYBTwvrufG7H8XODgx2g7AZ+6+wVmdgXwi/D0QuAGd18Svs1aoAAoA0rdPb1Kj1Yald37S/jHh2v52wdr2LmvhBP6dOBPlx3Lt/p0UFmIfE1HLQ0ziwGmAKcDucA8M8tw9+URy6QCtwMnuvtOM+sUsYoHgTjg+sj1uvvJEbd/EZgRvroGOCW8nrOAJ4CRETc91d23V+ExSiOzvfAAf31/Df/8aB2FB0r5dv9O/GhsX4antAs6mki9F82Wxggg291zAMxsGnA+sDximeuAKe6+E8Ddtx6c4e6zzGzMkVZuZgnAWODq8PIfRsz+GEiO6pFIo7d5934efy+HafPWc6C0nLOP6cKPxvRlYNfWQUcTaTCiKY1uwIaI67l88S9/gDQAM/uA0BDW3e7+RpQZxgOz3H1PJfOuBV6PuO7AW2bmwOPu/kRlKzSzScAkgJSUlChjSH21Ln8vj723mn8vyMUdLhjWjRvG9KFPx/igo4k0ONGURmWDv17JelKBMYS2DOaa2WB33xXF+icCT33pTs1OJVQaJ0VMPtHdN4WHv942sxXuPudL4UJl8gRAenp6xazSQKzMK+CRd7PJWLKJ2JgmTDg+hUmje9O9fVzQ0UQarGhKIxfoHnE9GdhUyTIfu3sJsMbMsgiVyLyvWrGZdSA0/DW+wvQhhIrkLHfPPzjd3TeFv241s5fDt/1SaUjDtjR3Nw+/u4o3M/OIaxbDD07uzQ9O6kWn1i2CjibS4EVTGvOAVDPrBWwEJgCXV1hmOqEthr+bWSKh4aqcKNZ9CTDT3YsOTjCzFOAl4Ep3XxkxvRXQxN0Lwt+PA+6N4j6kgfh0zQ4efjebOSu30bpFLDeP7cvVJ/aiXatmQUcTaTSOWhruXmpmk4E3Ce2veNrdM83sXmC+u2eE540zs+WEDoe97eAWQvjQ2v5AvJnlAte6+5vh1U8AKh6+eyfQAXgkfFjkwUNrk4CXw9NigalV2G8i9ZS7M2fVdqa8k82na3fQoVUzfn5mP64c1YOEFjqJoEhtM/eGPeSfnp7u8+fPDzqGVFF5ufPW8jwemZ3NZ7m76dKmBZNG92bC8Sm0bKbzQonUJDNbcKTPwekT4VKnlJaV8+rSzUx5N5uVeYX06BDH/Rcew4XDk2kWq/NCiQRNpSF1QnFpOS8tzOXR91azLn8faUnx/HnCsZxzTBdidRJBkTpDpSGB2l9cxrR563liTg6bdxcxJLkNj195HKcPSKKJTiIoUueoNCQQBUUl/PPjdfx17hry9xYzomd77r9oCKNTE3VeKJE6TKUhtWrn3mL+9sEa/v7hWvYUlTI6rSOTT+3LiF7tg44mIlFQaUit2LqniCfn5vDsJ+vZV1zGGYOS+NGpfRmS3DboaCJSBSoNqVEbduzj8TmreX5+LqVl5Zw3tCs3ntqXtKSEo99YROoclYbUiNXbCnl09mqmL9qIGVx8XDI/PKUPPTq0CjqaiHwDKg2pVss37WHK7GxeW7qZ5rFNuPJbPZg0ujdd2rQMOpqIVAOVhlSLJRt28dCsVcxasZX45rHccEofrjmpF4nxzYOOJiLVSKUh38j2wgM88PoKXliQS7u4pvzs9DS+d0JP2rTUeaFEGiKVhnwtpWXlPPvJev7wVhb7S8q4/pTe3DQ2lfjm+pESacj0Gy5VNn/tDv5rRiafb97DSX0Tufu8QfTtpP+SJ9IYqDQkatsKDvC71z/npYUb6dqmBY9cMZyzBnfWJ7hFGhGVhhxVaVk5z3y0jj++vZKi0jJuHNOHyWP7EtdMPz4ijY1+6+UrfZKTz10ZmazYUsDotI7c/Z2B9O6ooSiRxkqlIZXauqeI3772OdMXb6Jb25Y89t3jOGNQkoaiRBo5lYZ8QUlZOf/4cC1/+s8qikvLuWlsX24c01f/LU9EAJWGRPhodT53ZSxjZV4hp/bryF3fGUTPRJ32Q0QOU2kIW3YX8ZvXPueVJZtIbteSJ7+XzmkDOmkoSkS+RKXRiBWXlvO3D9bw0KxVlJQ7P/52KjeM6UOLphqKEpHKqTQaqQ+yt3PnjGWs3raX0wZ04s5zB5HSIS7oWCJSx6k0GpnNu/fz3zM/59Wlm0lpH8fT309nbP+koGOJSD2h0mgkikvLeer9HP4yK5tyd245PY1Jo3trKEpEqkSl0QjMWbmNuzMyydm+l3EDk/ivcwfSvb2GokSk6lQaDdjGXfu575XlvJG5hZ4d4vj71cczpl+noGOJSD2m0miADpSW8eScHB5+NxuAW8elcd3o3jSP1VCUiHwzKo0G5t2srdyTkcna/H2cNbgzd5wzgOR2GooSkeqh0mggNuzYx70zl/P28jx6J7bimWtGMDqtY9CxRKSBUWnUc0UlZTwxJ4cp72bTxIyfn9mPa0/qpaEoEakRKo167J0VedzzynLW5e/jnGO6cMc5A+jatmXQsUSkAVNp1EPr8/dxzyuZzFqxlT4dW/F/147kpNTEoGOJSCOg0qhHikrKeHT2ah59bzWxTYzbz+rP1Sf2ollsk6CjiUgjodKoB9yd/3y+lXtnZrJhx36+M7Qrd5w9gM5tWgQdTUQaGZVGHbd2+17ufiWT2VnbSO0Uz9TrRnJCHw1FiUgwVBp11P7iMh6Znc3j7+XQLLYJvz5nAFed0JOmMRqKEpHgRPUOZGZnmlmWmWWb2S+PsMylZrbczDLNbGrE9DfMbJeZzayw/FwzWxy+bDKz6eHpV5jZZ+HLh2Y2tCo56jt3541lWzjtf9/jL+9kc/YxnXnnZ6fwg5N7qzBEJHBH3dIwsxhgCnA6kAvMM7MMd18esUwqcDtworvvNLPIExw9CMQB10eu191Pjrj9i8CM8NU1wCnh9ZwFPAGMjCZHfZezrZC7X1nOnJXb6N85gX9NGsXI3h2CjiUickg0w1MjgGx3zwEws2nA+UDkm/V1wBR33wng7lsPznD3WWY25kgrN7MEYCxwdXj5DyNmfwwkVyFHvbSvuJSH38nmybk5tIiN4c5zB/K9b/UgVlsWIlLHRFMa3YANEddzgZEVlkkDMLMPgBjgbnd/I8oM44FZ7r6nknnXAq9XIQfhHJOASQApKSlRxqh97s7ry7bw3zOXs2l3ERcO78Yvz+pPpwQdFSUidVM0pWGVTPNK1pMKjCG0ZTDXzAa7+64o1j8ReOpLd2p2KqHSOKkKOUIT3Z8gNKxFenp6pcsELXtrIXdnZPJ+9nYGdGnNQxOHkd6zfdCxRES+UjSlkQt0j7ieDGyqZJmP3b0EWGNmWYRKZN5XrdjMOhAadhpfYfoQQkVylrvnVyFHnbf3QCkPvbOKp99fQ4umMdxz3iCuGJmioSgRqReiKY15QKqZ9QI2AhOAyyssM53QFsPfzSyR0HBVThTrvgSY6e5FByeYWV6SFWIAAAdtSURBVArwEnClu6+sYo46y92Z+dlmfvPq52zZU8QlxyXzi7P6kxjfPOhoIiJRO2ppuHupmU0G3iS0v+Jpd880s3uB+e6eEZ43zsyWA2XAbQe3EMxsLtAfiDezXOBad38zvPoJwP0V7vJOoAPwiJkBlLp7+pFyfKNHX0tW5RVwV0YmH67OZ1DX1ky5YjjH9WgXdCwRkSoz9zo55F9t0tPTff78+YHcd+GBUv78n5X87YO1tGoey61n9OPyESnENKls94yISN1gZgvcPb2yefpEeA1wdzKWbOI3r37O1oIDTDi+O7ed0Y8OGooSkXpOpVHNsrYUcOeMZXyyZgfHdGvD41cex7AUDUWJSMOg0qgme4pK+NPbq/jHR2tJaBHLb8YPZsLxGooSkYZFpfENuTsvL9rIb19bQf7eA0wckcJt4/rRrlWzoKOJiFQ7lcY3sHzTHu7KWMa8tTsZ2r0tT38/nSHJbYOOJSJSY1QaX8Pu/SX88e2VPPPRWtq0bMr9Fx7DpendaaKhKBFp4FQaVVBe7ry4MJcH3lhB/t5irhiZwq3j+tE2TkNRItI4qDSitGzjbu6csYyF63cxLKUtf796BIO7tQk6lohIrVJpHMXufSX8/q0snv1kHe3imvE/Fw/h4uHJGooSkUZJpXEE5eXOvxfkcv8bK9i1r5grR/XgltP70SauadDRREQCo9KoxO79JVz19Kcs3rCL9B7tuOf8EQzqqqEoERGVRiVat4ilZ4c4rhzVgwuHdyN84kQRkUZPpVEJM+NPE4YFHUNEpM7Rf/4REZGoqTRERCRqKg0REYmaSkNERKKm0hARkaipNEREJGoqDRERiZpKQ0REombuHnSGGmVm24B1X/PmicD2aoxTXZSrapSrapSrahpirh7u3rGyGQ2+NL4JM5vv7ulB56hIuapGuapGuaqmseXS8JSIiERNpSEiIlFTaXy1J4IOcATKVTXKVTXKVTWNKpf2aYiISNS0pSEiIlFTaYiISNRUGoCZ9TOzxRGXPWb2EzNrb2Zvm9mq8Nd2AWT7qZllmtkyM3vOzFqYWS8z+ySc619m1iyAXD8OZ8o0s5+EpwXyfJnZ02a21cyWRUyrNIuFPGRm2Wb2mZkNr+Vcl4Sfs3IzS6+w/O3hXFlmdkYt53rQzFaEn5OXzaxtHcl1XzjTYjN7y8y6hqcH+jpGzLvVzNzMEutCLjO728w2RryXnR0xr3peR3fXJeICxABbgB7A/wC/DE//JfBALWfpBqwBWoavPw98P/x1QnjaY8ANtZxrMLAMiCP03x//A6QG9XwBo4HhwLKIaZVmAc4GXgcMGAV8Usu5BgD9gNlAesT0gcASoDnQC1gNxNRirnFAbPj7ByKer6BztY74/mbgsbrwOoandwfeJPTh4cS6kAu4G7i1kmWr7XXUlsaXfRtY7e7rgPOBf4Sn/wO4IIA8sUBLM4sl9Ca9GRgL/DvAXAOAj919n7uXAu8B4wno+XL3OcCOCpOPlOV84BkP+Rhoa2ZdaiuXu3/u7lmVLH4+MM3dD7j7GiAbGFGLud4Kv5YAHwPJdSTXnoirrYCDR+4E+jqG/RH4eUSmupKrMtX2Oqo0vmwC8Fz4+yR33wwQ/tqpNoO4+0bg98B6QmWxG1gA7Ir4Bc8ltEVSm5YBo82sg5nFEfrrqjsBP18VHClLN2BDxHJBPH+VqUu5riH01zLUgVxm9hsz2wBcAdxZF3KZ2XnARndfUmFW4M8XMDk8NPZ0xBBxteVSaUQI7xs4D3gh6CwA4Rf8fEKbk10J/aV1ViWL1upx0+7+OaEhjLeBNwht9pZ+5Y3qDqtkWl047rxO5DKzOwi9ls8enFTJYrX983aHu3cnlGlyeHJgucJ/KN3B4QL7wuxKptXm8/Uo0Ac4ltAfmn8IT6+2XCqNLzoLWOjueeHreQc3LcNft9ZyntOANe6+zd1LgJeAEwht8saGl0kGNtVyLtz9r+4+3N1HE9pEXkXwz1ekI2XJJbRVdFAgz18lAs9lZlcB5wJXeHggvC7kijAVuCj8fZC5+hD6Q26Jma0N3/dCM+sccC7cPc/dy9y9HHiSw0NQ1ZZLpfFFEzk8NAWQAVwV/v4qYEYt51kPjDKzODMzQvtblgPvAhcHmAsz6xT+mgJcSOh5C/r5inSkLBnA98JHuYwCdh8cxgpYBjDBzJqbWS9CBxZ8Wlt3bmZnAr8AznP3fXUoV2rE1fOAFRG5Ankd3X2pu3dy957u3pPQG/Jwd98SZC449AfSQeMJDSVDdb6ONbVnv75dCO1kzgfaREzrAMwi9Ff0LKB9ALnuIfSLsgz4J6GjH3qHX/BsQkNpzQPINZdQgS0Bvh3k80WosDYDJYR+ga89UhZCm+lTCB09spSII5hqKdf48PcHgDzgzYjl7wjnygLOquVc2YTGvBeHL4/VkVwvhn/2PwNeAbrVhdexwvy1HD56Kuifr3+G7/czQkXRpbpfR51GREREoqbhKRERiZpKQ0REoqbSEBGRqKk0REQkaioNERGJmkpDRESiptIQEZGo/T+CLZ3hovLJ+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# # plot CV误差曲线\n",
    "test_means = grid_search_min_child_samples_v2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search_min_child_samples_v2.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid_search_v1.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid_search_v1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "x_axis = min_child_samples_s\n",
    "\n",
    "plt.plot(x_axis, -test_means)\n",
    "# #plt.errorbar(x_axis, -test_scores, yerr=test_stds ,label = ' Test')\n",
    "# #plt.errorbar(x_axis, -train_scores, yerr=train_stds,label =  +' Train')\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done   4 out of  12 | elapsed: 151.0min remaining: 302.1min\n",
      "[Parallel(n_jobs=6)]: Done   7 out of  12 | elapsed: 250.7min remaining: 179.1min\n",
      "[Parallel(n_jobs=6)]: Done  10 out of  12 | elapsed: 282.8min remaining: 56.6min\n",
      "[Parallel(n_jobs=6)]: Done  12 out of  12 | elapsed: 285.3min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      class_weight=None, colsample_bytree=0.7,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=64,\n",
       "                                      max_depth=7, min_child_samples=20,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=9431,\n",
       "                                      n_jobs=6, num_leaves=30,\n",
       "                                      objective='regression', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.7,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='warn', n_jobs=6,\n",
       "             param_grid={'min_child_samples': range(50, 90, 10)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':30,\n",
    "          'max_bin': 64, # 稀疏离散特征一共有167维，先不设置很大，提高训练速度\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "#           'early_stopping_round':10, # 添加 early_stopping_round，减少迭代次数\n",
    "#           'metric':'auc'\n",
    "         }\n",
    "lg_v1 = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "min_child_samples_s = range(50,90,10) \n",
    "tuned_parameters = dict( min_child_samples = min_child_samples_s)\n",
    "\n",
    "grid_search_min_child_samples_v3 = GridSearchCV(lg_v1, n_jobs=6,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search_min_child_samples_v3.fit(train_X , train_Y)\n",
    "# grid_search_v1.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(model_path + 'grid_search_min_child_samples_v3.pkl','wb') as fw:\n",
    "        pk.dump(grid_search_min_child_samples_v3,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump grid_search_min_child_samples_v3 error,',e)\n",
    "try:\n",
    "    # 序列化到磁盘中\n",
    "    with open(model_path + 'lg_v3.pkl','wb') as fw:\n",
    "        pk.dump(lg_v1,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg_v1 error,',e)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f73b6b23ef0>]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAD4CAYAAAApWAtMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxV1bn/8c+XhBCGMIcZZEpAVMaIVkUFRLH1itai4FDb2jpUnK9XvUNvr/56r62zFbFWrW0VZ0WKFURExQElDKKgzAhhDPOY+fn9cXb0cEjIgZDsnOR5v155wVl77bWfnQ15stbeey2ZGc4551xl1As7AOecc4nPk4lzzrlK82TinHOu0jyZOOecqzRPJs455yotOewAwtK6dWvr2rVr2GE451xCmTt37hYzS48tr7PJpGvXrmRnZ4cdhnPOJRRJ35ZV7sNczjnnKs2TiXPOuUrzZOKcc67SPJk455yrNE8mzjnnKs2TiXPOuUqLK5lIGilpiaTlku4sp87FkhZLWiRpYlT5VEk7JE2JqT9L0oLga72kSUH5KEkLg/JsSacF5UOj6i+QlCfpgmDbs5JWRW3rf6TfEOecc4evwvdMJCUB44ERQA4wR9JkM1scVScDuAs41cy2S2oT1cR9QCPgmuh2zWxI1P6vAW8GH2cAk83MJPUFXgZ6m9lMoH9QvyWwHHgnqsnbzezV+E7bOefqnrzCYu59+xt+OaQbnVo0Oqptx9MzGQwsN7OVZlYAvAiMiqnzK2C8mW0HMLPNpRvMbAawu7zGJaUBw4BJQf099v0iK42BshZc+QnwtpntiyN+55xzwHOzv+XZT1azbvv+o952PMmkI7A26nNOUBYtE8iU9LGk2ZJGHkYMFwIzzGxXaYGkCyV9A7wF/KKMfcYAL8SU/S4YHntIUoOyDiTp6mDoLDs3N/cwQnTOucS2N7+ICe+v4LSerTmpe6uj3n48yURllMX2FpKBDOBMYCzwlKTmccYwlpjEYGZvmFlv4ALgngOCkdoDJwDToorvAnoDJwItgTvKOpCZPWlmWWaWlZ5+0NQyzjlXaz37yWq27i3g1rMzq6T9eJJJDtA56nMnYH0Zdd40s0IzWwUsIZJcDklSKyLDaG+Vtd3MPgR6SGodVXwx8IaZFUbV22AR+cBfgjadc84Bu/IKefLDlQzr3YaBXVpUyTHiSSZzgAxJ3SSlEBlimhxTZxIwFCD4wZ8JrIyj7dHAFDPLKy2Q1FOSgr8PBFKArVH7HNSTCXorBPtdAHwVx7Gdc65OeHrWKnbuL+TWEVXTK4E4nuYysyJJ44gMKyUBz5jZIkl3A9lmNjnYdrakxUAxkSertkLkEWAiQ1BNJOUAV5lZ6RDVGODemENeBPxUUiGwH7ik9Ia8pK5EekkfxOzzvKR0IkNyC4BrD+N74JxztdaOfQU889EqRh7XjuM7Nquy4+j7B6fqlqysLPMp6J1ztd0fpn7DhA9WMPWm0+nVLq3S7Umaa2ZZseX+BrxzztVSW/bk85ePV3Ne3w5HJZEciicT55yrpZ54fwX5RcXcfFaFz0NVmicT55yrhTbtyuPvs7/lwgGd6JHepMqP58nEOedqofEzl1NcYtw0vOp7JeDJxDnnap2c7ft44fM1jM7qTJdWR3cOrvJ4MnHOuVrmsfeWI8QNw3pW2zE9mTjnXC2yesteXpmbw6UndaFD84bVdlxPJs45V4s8OmMZyfXEr8/sUa3H9WTinHO1xPLNe5i0YB1XntKVNk1Tq/XYnkycc66WePjdpaTWT+Ka07tX+7E9mTjnXC3w9YZdTFm4gZ+f2pVWTcpc0qlKeTJxzrla4KHpS0lLTebqIdV7r6SUJxPnnEtwC3N28M7iTfzytO40a1Q/lBg8mTjnXIJ7cPpSmjeqzy9O6xpaDJ5MnHMugc39dhvvL8nlmtN7kJYaTq8EPJk451xCe+CdpbRuksKVpxwTahyeTJxzLkF9umIrn6zYynVn9qRRSoUL51apuJKJpJGSlkhaLunOcupcLGmxpEWSJkaVT5W0Q9KUmPqzJC0IvtZLmhSUj5K0MCjPlnRa1D7FUftMjirvJukzScskvRSsVe+cc7WWmfHg9CW0bdqAy07qEnY4Fa8BLykJGA+MAHKAOZImm9niqDoZwF3AqWa2XVKbqCbuAxoB10S3a2ZDovZ/DXgz+DgDmGxmJqkv8DKRNeQB9ptZ/zLC/D3wkJm9KOkJ4CpgQkXn5pxzierDZVuYs3o794w6jtT6SWGHE1fPZDCw3MxWmlkB8CIwKqbOr4DxZrYdwMw2l24wsxnA7vIal5QGDAMmBfX32PcL0zcGDrlIvSQF+78aFP0VuCCO83LOuYRkZjz4zhI6Nm/IxSd2DjscIL5k0hFYG/U5JyiLlglkSvpY0mxJIw8jhguBGWa2q7RA0oWSvgHeAn4RVTc1GPqaLak0YbQCdphZ0SHiK2336mD/7Nzc3MMI0Tnnao53v97MFzk7uXF4Txokh98rgfiSicooi+0tJAMZwJnAWOApSc3jjGEs8MIBjZu9YWa9ifQw7ona1MXMsoBLgYcl9YgzvtJ2nzSzLDPLSk9PjzM855yrOUpKjAenL6Vrq0b8eGCnsMP5TjzJJAeI7kd1AtaXUedNMys0s1XAEiLJ5ZAktSIyjPZWWdvN7EOgh6TWwef1wZ8rgfeBAcAWoLmk0vs/ZcXnnHO1wtRFG/l6wy5uOiuD+kk154HceCKZA2QET0ylAGOAyTF1JgFDAYIf/JnAyjjaHg1MMbO80gJJPYP7IEgaCKQAWyW1kNQg6hinAouD+yszgZ8ETVzJ9zfznXOu1igOeiU92zTh/H5ljuaHpsJkEtyLGAdMA74GXjazRZLulnR+UG0akR/4i4n8YL/dzLZC5BFg4BVguKQcSedENT+GmCEu4CLgK0kLiDxFdkmQMI4FsiV9ERzj3qgnyu4AbpW0nMg9lKcP79vgnHM13z++WM/yzXu45axMkuqVNcIfHn3/4FTdkpWVZdnZ2WGH4ZxzcSkqLuGsBz8gtX4S/7xxCPVCSiaS5gb3rg9QcwbcnHPOlev1eetYvXUft53dK7REciieTJxzroYrKCrhkRnL6NepGWcd26biHULgycQ552q4l7LXsm7Hfm4ZkUnwfFKN48nEOedqsLzCYh57bxlZx7TgjMya+36cJxPnnKvBnv9sDZt25XPr2TW3VwKeTJxzrsbaV1DEhPeXc0qPVpzSo3XY4RySJxPnnKuh/vbpt2zZU8BtZ2eGHUqFPJk451wNtDuvkCc+WMEZmekMOqZl2OFUyJOJc87VQH/5eDU79hUmRK8EPJk451yNs3NfIX+etZIRfdrSt1O8E7CHy5OJc87VMH+etZLdeUXcOiIxeiXgycQ552qUrXvy+cvHq/hR3/Yc275p2OHEzZOJc87VIH/6cCX7C4u55awKl4SqUTyZOOdcDbF5dx5/+3Q1F/TvSM82aWGHc1g8mTjnXA3x+MwVFBYbNw5PrF4JeDJxzrkaYf2O/Uz8bA0/GdiJrq0bhx3OYYsrmUgaKWmJpOWS7iynzsWSFktaJGliVPlUSTskTYmpP0vSguBrvaRJQfkoSQuD8mxJpwXl/SV9GrS/UNIlUW09K2lVVHv9j+Sb4ZxzYXls5nIM44bhPcMO5YgkV1RBUhKR5XNHADnAHEmTo5bMRVIGcBdwqpltlxQ94f59QCPgmuh2zWxI1P6v8f267TOAyWZmkvoCLwO9gX3AT81smaQOwFxJ08xsR7Df7Wb26uGcvHPO1QRrtu7j5TlrGTu4C51aNAo7nCMST89kMLDczFaaWQHwIjAqps6vgPFmth3AzDaXbjCzGcDu8hqXlAYMAyYF9ffY92sJNwYsKF9qZsuCv68HNgM1dz5m55yL06PvLSOpnhg3LDF7JRBfMukIrI36nBOURcsEMiV9LGm2pJGHEcOFwAwz21VaIOlCSd8AbwG/iN1B0mAgBVgRVfy7YPjrIUkNyjqQpKuDobPs3NzcwwjROeeqxsrcPbw+L4fLTz6Gtk1Tww7niMWTTMqaQN9iPicDGcCZwFjgKUnxzgEwFnjhgMbN3jCz3sAFwD0HBCO1B/4O/NzMSoLiu4gMhZ0ItATuKOtAZvakmWWZWVZ6undqnHPhe/jdZTRITuK6M3uEHUqlxJNMcoDOUZ87AevLqPOmmRWa2SpgCZHkckiSWhEZRnurrO1m9iHQQ1LroH7ToO5/mtnsqHobLCIf+EvQpnPO1WhLNu7mHwvX87NTu9K6SZkDKgkjnmQyB8iQ1E1SCjAGmBxTZxIwFCD4wZ8JrIyj7dHAFDPLKy2Q1FPBcmKSBhIZztoaHPsN4G9m9kp0I0FvhWC/C4Cv4ji2c86F6qHpS2mckszVQ7qHHUqlVfg0l5kVSRoHTAOSgGfMbJGku4FsM5scbDtb0mKgmMiTVVsh8ggwkSGoJpJygKvMbFrQ/Bjg3phDXgT8VFIhsB+4JHiy62LgdKCVpJ8FdX9mZguA5yWlExmSWwBce0TfDeecqyZfrdvJ1EUbuWl4Bi0ap4QdTqXp+wen6pasrCzLzs4OOwznXB111bNzyP52O7PuGErT1PphhxM3SXPNLCu23N+Ad865ajZvzXZmfLOZq0/vnlCJ5FA8mTjnXDV7aPpSWjZO4WendA07lKPGk4lzzlWjz1ZuZdayLVx3Rg8aN6jwtnXC8GTinHPVxMx4YPpS0tMacPnJx4QdzlHlycQ556rJx8u38vmqbYwb2pOGKUlhh3NUeTJxzrlqYGbc/84SOjRLZczgzhXvkGA8mTjnXDWYuWQzC9bu4IbhGTRIrl29EvBk4hKcmbFzXyE52/eFHYpz5TIzHnhnKV1aNuIngzqFHU6VqD2PErhax8zYtreADTvz2Lgzjw278ti4c/93nzfuzGPDzjz2FxYDcOOwntx6dq+Qo3buYNMWbWTR+l3cP7of9ZNq5+/wnkxcKIpLjK178tkQJISNO/cHySLvgGRRUFxywH5J9US7pqm0a5bKsR2aMqx3G9o1S2X+mh08+t5y+nVuzvBj24Z0Vs4drLjEeHD6UrqnN+aC/h3CDqfKeDJxR11hcQmbd+cf0IvYsDOPjbu+TxKbduVRVHLgVD4pSfVo1yySKAZ0aU67Zqm0b5pKu2YNad8slfbNUmnVpAFJ9Q5eFSGvsJjVW/dyy0sLeOvGIXRumZir1bnaZ8rC9SzdtIdHxw4guZb2SsCTiTtM+UXFbNqZz4ad+9m4Ky8qWez/Lmnk7skndsq31Pr16NCsIe2apXJS95a0bxYkiaCX0b5ZKi0bpxBMGH3YUusnMeGyQZz3x1lc+9xcXrvuFFLr176bnC6xFBWX8Mi7y+jVNo3zTmgfdjhVypOJ+87+guIDkkIkWew/YOhp696Cg/ZLa5D8XY+iV7u073oSpUmifdOGNG2YfMSJIl5dWjXioUv6c9Vfs/nt5EXce1HfKj2ecxWZtGA9K7fs5YnLB1GvjB51beLJpI7YnVd4QFKIJIsDh6F27i88aL/mjerTrmkkKfTt1PzAJNEslbZNU0mrQRPVDT+2LdcP7cH4mSsYeEwLLs6qfc/zu8RQWFzCIzOWcnzHppxzXO2/j+fJJMGZGTv3Fx6YJErvVUQNQ+3JLzpo39ZNUmjXLJVOLRpxYteW3yWJyJ8Nadc0NSHf0r11RC8WrN3Bf036iuM6NOW4Ds3CDsnVQa9k57B2237u/tnxVd4rrwl8PZMarKTE2Lav4OAkETMMlVd44BNPErRJa3DQPYnSJNG+WSptmjaolS9OldqyJ5/zHv2IlOR6/OOG02jWsOb0nlztl1dYzND736dds1Rev+6UWpVMylvPxHsmISkuMbYEj8YelCR25rFh13427cw/6NHY5HqibZAg+nRoyvDg0dj2wc3t9s1SSU9rUGufZY9X6yYNGH/ZQC7506fc9vICnrwiq9aPWbua48XP17BhZx73j+5XqxLJocSVTCSNBB4hsmzvU2YWu9QuwbK6vwUM+MLMLg3KpwInAx+Z2XlR9WcBacHHNsDnZnaBpFHAPUAJUATcbGYfBftcCfxnsM//M7O/BuWDgGeBhsA/gZssxC5XuY/GRj31tGl3PsWHeDR2YJcWh/VorDvYoGNa8B8/Opb/+cdinvhwBb8+s2fYIbk6YH9BMePfX8FJ3VpySo9WYYdTbSpMJpKSgPHACCAHmCNpspktjqqTAdwFnGpm2yW1iWriPqARcE10u2Y2JGr/14A3g48zgMnBuu99gZeB3pJaAv8NZBFJWHODOLYDE4CrgdlEkslI4O34vw3xO9JHYxvWT/puqOnkHq2O+qOxrmw/O6Urc7/dzv3TltC/c3NO6dE67JBcLff32avJ3Z3P+EsH1qn/z/H0TAYDy81sJYCkF4FRwOKoOr8Cxgc/2DGzzaUbzGyGpDPLa1xSGjAM+HlQf0/U5sZEEgfAOcB0M9sW7DcdGCnpfaCpmX0alP8NuIAqSib//eYiXpyz9oCymvJorDuYJH5/UV++2bibG1+Yz5QbhtCuWWrYYblaak9+EU98sJIhGa0Z3K1l2OFUq3iSSUcg+qdnDnBSTJ1MAEkfExkK+62ZTY0zhguBGWa2q7RA0oXA/xEZ/vrRIeLoGHzllFF+EElXE+nB0KVLlzjDiwl2QEcGHtOixj4a6w7WuEEyT1w+kPMf+5hxE+fxwtUn1/l7Sq5qPPvxKrbtLeC2OjhHXDz/o8r6dTr2fkQykAGcCYwFnpLUPM4YxgIvHNC42Rtm1ptID+OeCuKIJ77Sdp80sywzy0pPT48zvAOd1L0VF2d1ZkhGOj3bpHkiSRA926Rx70V9yf52O/e+/U3Y4bhaaOf+Qp78cCVnHduG/p3j/fFXe8STTHKA6De/OgHry6jzppkVmtkqYAmR5HJIkloRGUZ7q6ztZvYh0ENS60PEkRP8/VDxOcf5/Trws1O68vRHq3hr4Yaww3G1zNMfrWJXXhG3jMgMO5RQxJNM5gAZkrpJSgHGAJNj6kwChgIEP/gzgZVxtD0amGJmeaUFknoquLkgaSCQAmwFpgFnS2ohqQVwNjDNzDYAuyWdHOz3U76/me/cAf79h8cysEtz/u3VL1i+eU/FOzgXh+17C3jmo1Wce3y7OvuSbIXJxMyKgHFEfph/DbxsZosk3S3p/KDaNGCrpMXATOB2M9sK3z0C/AowXFKOpHOimh9DzBAXcBHwlaQFRJ4iu8QithEZ8poTfN1dejMeuA54ClgOrKCKbr67xJeSXI/xlw2kQf0krntuLnvLmBnAucP1pw9Xsreg7vZKwN+ADzsMF5KPl2/hiqc/47y+HXhkTH9/0s4dsdzd+Zz+h5mcfVxbHhkzIOxwqlx5b8D7Iy2uTjq1Z2tuO7sXk79Yz99nfxt2OC6BTXh/BflFxdw0vMLbxLWaJxNXZ113Rg+G927DPVMWM2/N9rDDcQlo4848nvvsWy4a2Inu6U3CDidUnkxcnVWvnnjw4v60bZrK9c/PY+ue/LBDcgnmsZnLKCkxbqzjvRLwZOLquGaN6vPE5YPYureAm19acNB8ac6VZ+22fbw0Zy2XnNjZl4nGk4lzHN+xGfeMOo5Zy7bwyLtLww7HJYg/vrcMSYwb5hOIgicT5wC45MQujB7UiUffW85732wKOxxXw63espfX5q3j0sFdaN+sYdjh1AieTJwL3HPB8Rzbvim3vPQFa7ftCzscV4M9MmMZ9ZPEr4f2CDuUGsOTiXOB1PpJPHH5QErM+PXz88grLA47JFcDLdu0m0kL1nHlD7rSJs1noC7lycS5KMe0asyDF/fny3U7+Z9/LK54B1fnPPzuMhrVT+KaM7xXEs2TiXMxRvRpy3Vn9uCFz9fwSvbaindwdcbi9bt468sN/OK0brRsnBJ2ODWKJxPnynDbiEx+0L0V/znpKxav31XxDq5OeHD6UtJSk/nlad3DDqXG8WTiXBmSk+rx6NgBNG9Un+uen8vO/YVhh+RC9sXaHbz79SauHtKdZo18HaNYnkycK0d6WgPGXzqQddv386+vfEFdnRTVRTwwfSktGtXn56d1CzuUGsmTiXOHkNW1JXf98FimL97Enz6MZ4keVxtlr97Gh0tzufaMHjRpEM9q53WPJxPnKvCLU7vyoxPa84ep3/Dpiq1hh+NC8MA7S2ndpAE//UHXsEOpsTyZOFcBSfz+J33p2roxN7wwn0278ireydUanyzfwqcrt/LrM3vQMCUp7HBqrLiSiaSRkpZIWi7pznLqXCxpsaRFkiZGlU+VtEPSlJj6syQtCL7WS5oUlF8maWHw9YmkfkF5r6j6CyTtknRzsO23ktZFbfvhkX5DnCtLkwbJ/OnyQewrKGLcxHkUFpeEHZKrBmbGA9OX0q5pKpee1CXscGq0CpOJpCQiy+eeC/QBxkrqE1MnA7gLONXMjgNujtp8H3BFbLtmNsTM+ptZf+BT4PVg0yrgDDPrS2SZ3ieD+kui6g8C9gFvRDX5UOl2M/tnHOfu3GHJaJvG//34BOas3s7v3/4m7HBcNXh/aS5zv93OuGE9Sa3vvZJDiadnMhhYbmYrzawAeBEYFVPnV8B4M9sOYGabSzeY2Qxgd3mNS0oDhgGTgvqflLYDzAY6lbHbcGCFmfkSea5ajerfkSt/cAxPfbSKf365IexwXBUyMx6avpROLRpycVbnsMOp8eJJJh2B6NeAc4KyaJlApqSPJc2WNPIwYrgQmGFmZb0ZdhXwdhnlY4AXYsrGBUNjz0hqUdaBJF0tKVtSdm5u7mGE6Nz3/uNHfejfuTn/9upCVuTuCTscV0WmL97Ewpyd3Dg8g5Rkv71ckXi+QyqjLPaB+2QgAzgTGAs8Jal5nDGM5eDEgKShRJLJHTHlKcD5wCtRxROAHkB/YAPwQFkHMrMnzSzLzLLS09PjDM+5A6Uk1+PxywaSklyP656by76CorBDckdZSYnx4PSldGvdmB8PiP3d2ZUlnmSSA0T38ToB68uo86aZFZrZKmAJkeRySJJaERlGeyumvC/wFDDKzGKfxTwXmGdm3y06YWabzKzYzEqAPwdtOldlOjRvyCNj+rNs8x7+/fUv/YXGWuafX23gm427ufmsDJKTvFcSj3i+S3OADEndgl7BGGByTJ1JwFAASa2JDHvF84bXaGCKmX33rKWkLkRuxl9hZmUte3dQT0ZS+6iPFwJfxXFs5yplSEY6t56VyaQF63nuszVhh+OOkuKSyL2SjDZNOK9vh7DDSRgVJhMzKwLGAdOAr4GXzWyRpLslnR9UmwZslbQYmAncXtqjkDSLyJDUcEk5ks6Jar6sex+/AVoBjweP+WaXbpDUCBjB909+lfqDpC8lLSSS1G6J5+Sdq6zrh/ZkaK907v7HIhas3RF2OO4oeHPBOlbk7uWWEZkk1StrlN+VRXW1e56VlWXZ2dkVV3SuAjv2FXDeHz+ipMSYcuMQn5o8gRUWl3DWgx/QOCWZKTecRj1PJgeRNNfMsmLLfTDQuUpq3iiFCZcNYsueAm56cT7FJXXzF7Ta4LW5OXy7dR+3jsj0RHKYPJk4dxSc0KkZ/zPqOGYt28KjM5aFHY47AvlFxfzxveX069yc4ce2CTuchOPJxLmjZMyJnfnJoE48+t4yZi7ZXPEOrkZ5ec5a1u3Yz20jMpG8V3K4PJk4d5RI4p5Rx9OrbRq3vLSAtdv2hR2Si1NeYaRXcmLXFgzJaB12OAnJk4lzR1HDlCSeuHwQxcXG9RPnkV9UHHZILg7Pzf6Wzbvzue3sXt4rOUKeTJw7yrq2bswDF/djYc5O7v7H4rDDcRXYm1/EhPdXcGrPVpzcvVXY4SQsTybOVYGzj2vHtWf04PnP1vDa3Jyww3GH8NdPV7N1bwG3jugVdigJzZOJc1XkX8/O5OTuLfmPSV/y9Yay5jF1YduVV8ifPljJ0F7pDDqmzPlhXZw8mThXRZKT6vHo2AE0Ta3Pdc/NZVdeYdghuRjPfLSKnfsLvVdyFHgyca4KtUlLZfxlA1m7fT+3v/KFTwhZg+zYV8DTs1ZxznFtOaFTs7DDSXieTJyrYid2bcld5/Zm2qJN/HlWPPOfuurw5Icr2VNQxC0jMsMOpVbwZOJcNbjqtG788IR2/H7qEmavjF1VwVW3rXvyefaT1ZzXtwO92zUNO5xawZOJc9VAEr+/qC/HtGzEuInz2bwrr+KdXJV54oMV5BUWc/NZFS675OLkycS5apKWWp8Jlw9ib34R4ybOp7C4JOyQ6qRNu/L426ffcsGAjvRIbxJ2OLWGJxPnqlGvdmn8349P4PPV27hv2pKww6mTHp+5nOIS46bh3is5mjyZOFfNLhjQkStOPoYnP1zJ1K82hB1OnbJux35e+Hwto7M6cUyrxmGHU6vElUwkjZS0RNJySXeWU+diSYslLZI0Map8qqQdkqbE1J8VrKS4QNJ6SZOC8sskLQy+PpHUL2qf1cGKirErMLaUNF3SsuBPf/vI1Wj/ed6x9OvcnH99ZSErc/eEHU6d8dh7keUBxg3zXsnRVmEykZQEjAfOBfoAYyX1iamTAdwFnGpmxwE3R22+D7gitl0zG2Jm/c2sP/Ap3y/Fuwo4w8z6AvcAT8bsOjTYL3qlrzuBGWaWAcwIPjtXYzVITuLxywZSP0lc99w89hUUhR1Srfft1r28kp3D2MGd6di8Ydjh1Drx9EwGA8vNbKWZFQAvAqNi6vwKGG9m2wHM7LvFHMxsBrC7vMYlpQHDgElB/U9K2wFmA53iiHEU8Nfg738FLohjH+dC1bF5Qx4ZM4Clm3fzH2985S80VrFHZiwjqZ64fmjPsEOpleJJJh2BtVGfc4KyaJlApqSPJc2WNPIwYriQSK+irMmLrgLejvpswDuS5kq6Oqq8rZltAAj+LHOZNElXS8qWlJ2bm3sYITpXNU7PTOfm4Zm8MX8dz3+2Juxwaq3lm/cwaf46fvqDY2jTNDXscGql5DjqlDW5f+yvUMlABnAmkZ7ELEnHm9mOONofCzx10EGloUSSyWlRxaea2XpJbYDpkr4xsw/jOEYkaLMnCYbNsrKy/NdAVyPcMKwn89du5+5/LOaEjs3o17l52CHVOg+/u5TU+klce0aPsNF6sUMAABYeSURBVEOpteLpmeQAnaM+dwLWl1HnTTMrNLNVwBIiyeWQJLUiMoz2Vkx5XyIJZpSZffe6sJmtD/7cDLwR7AuwSVL7YN/2gK+Z6hJGvXrioYv7k57WgF8/P4/tewvCDqlW+WbjLqYs3MDPTulKqyYNwg6n1oonmcwBMiR1k5QCjAEmx9SZBAwFkNSayLBXPJMQjQammNl3rwNL6kLkZvwVZrY0qrxxcH8FSY2Bs4Gvgs2TgSuDv18JvBnHsZ2rMVo0TmHC5QPJ3Z3PTS8toLjEO85Hy0PTl5LWIJmrT+8edii1WoXJxMyKgHHANOBr4GUzWyTpbknnB9WmAVslLQZmAreX9igkzQJeAYZLypF0TlTzY4AXYg75G6AV8HjMI8BtgY8kfQF8DrxlZlODbfcCIyQtA0YEn51LKH07Nee/z+/Dh0tz+WPwCKurnC9zdjJt0SauGtKN5o1Swg6nVlNdfYIkKyvLsrOzK67oXDUyM2575QvemL+OZ38+mDMy08MOKaH9/C+fM3/tDj78t6E0Ta0fdji1gqS5Ma9mAP4GvHM1iiR+d8EJ9Gqbxk0vzidn+76wQ0pYc7/dzswluVx9endPJNXAk4lzNUzDlCQmXD6I4mLj+ufnkV9UHHZICenB6Uto1TiFK3/QNexQ6gRPJs7VQN1aN+a+0f34Imcn90xZHHY4CefTFVv5ePlWrjuzB40bxPMGhKssTybO1VAjj2/HNad357nZa3hjfk7Y4SQMM+PB6Uto27QBl598TNjh1BmeTJyrwW4/pxeDu7Xkrte/5JuNZU0S4WLNWraFOau3M25oT1LrJ4UdTp3hycS5Giw5qR6PXTqAtNT6XPfcPHbnFYYdUo1mZjwwfSkdmzfk4hM7V7yDO2o8mThXw7VJS+WxsQNYs20ft7+y0CeEPIQZX2/mi7U7uGFYTxoke6+kOnkycS4BnNS9FXeO7M3URRt5+qNVYYdTI5WUGA9OX8oxrRpx0aB4Jht3R5MnE+cSxC+HdGPkce34v7e/4fNV28IOp8aZumgjizfs4qbhGdRP8h9t1c2/484lCEncN7ovXVo24vqJ89i8O6/ineqI4hLjoelL6ZHemFH9Y1fIcNXBk4lzCSQttT4TLh/I7rxCxk2cT1FxSdgh1QhTFq5n2eY93DIik6R6Za2a4aqaJxPnEkzvdk35vx+fwOertnHfO0vCDid0RcUlPPzuMnq3S+OHx7cPO5w6y5OJcwnowgGduPzkLvzpg5VM/Wpj2OGE6vX561i1ZS+3jsiknvdKQuPJxLkE9V/n9aFfp2bc/soXrNqyN+xwQlFQVMIj7y6jb6dmjOjTNuxw6jRPJs4lqAbJSYy/bCBJSeK65+ayv6DuTQj5cvZa1u3Yzy0jMpG8VxImTybOJbBOLRrx8CX9WbJpN/8x6cs69UJjXmExj723nEHHtOBMX/cldJ5MnEtwZ/Zqw03DM3h93jpe+Hxt2OFUm4mfrWHjrjxu815JjRBXMpE0UtISScsl3VlOnYslLZa0SNLEqPKpknZImhJTf1awLO8CSeslTQrKL5O0MPj6RFK/oLyzpJmSvg6OcVNUW7+VtC6qvR8eyTfDuUR147AMTs9M57eTF7EwZ0fY4VS5fQVFPP7+Cn7QvRWn9GwddjiOOJKJpCRgPHAu0AcYK6lPTJ0M4C7gVDM7Drg5avN9wBWx7ZrZEDPrb2b9gU+B14NNq4AzzKwvcA/wZFBeBNxmZscCJwPXx8TxUGl7ZvbPis7LudqkXj3x8CX9SU9rwHXPzWPHvoKwQ6pSf/v0W7bsyee2szPDDsUF4umZDAaWm9lKMysAXgRGxdT5FTDezLYDmNnm0g1mNgPYXV7jktKAYcCkoP4npe0As4FOQfkGM5sX/H038DXgr7o6F2jZOIXHLxtI7u58bn5pASUltfP+yZ78Iv70wQpOz0wnq2vLsMNxgXiSSUcgeiA2h4N/iGcCmZI+ljRb0sjDiOFCYIaZlbVYw1XA27GFkroCA4DPoorHBUNjz0hqUdaBJF0tKVtSdm5u7mGE6Fxi6Ne5Ob/5lz68vySXx2YuDzucKvGXj1axfV8ht43wXklNEk8yKevOVuyvPMlABnAmMBZ4SlLzOGMYC7xw0EGloUSSyR0x5U2A14CboxLQBKAH0B/YADxQ1oHM7EkzyzKzrPR0f/rD1U6XndSFCwd05KF3l/Lh0tr1S9POfYU8OWslZx3bln6d4/0R46pDPMkkB4heZaYTsL6MOm+aWaGZrQKWEEkuhySpFZFhtLdiyvsCTwGjzGxrVHl9IonkeTMrvceCmW0ys2IzKwH+HLTpXJ0kid9deDyZbdK46cX5rNuxP+yQjpqnPlrJ7rwibvVeSY0TTzKZA2RI6iYpBRgDTI6pMwkYCiCpNZFhr5VxtD0amGJm301/KqkLkZvxV5jZ0qhyAU8DX5vZg9GNSIqekOdC4Ks4ju1crdUoJZkJlw+ksNj49fPzyC9K/Bcat+0t4JmPVvGjE9rTp0PTsMNxMSpMJmZWBIwDphG56f2ymS2SdLek84Nq04CtkhYDM4HbS3sUkmYBrwDDJeVIOieq+TEcPMT1G6AV8HjwmG92UH4qkafChpXxCPAfJH0paSGRpHbLYX0XnKuFuqc34f7Rffli7Q5+99bXYYdTaX/6YAX7Cou5+awKBz1cCFSX3piNlpWVZdnZ2RVXdC7B/e6txfx51ioeGdM/Ydf62Lw7j9P/MJNzj2/PQ5f0DzucOk3SXDPLii33N+Cdq+X+bWRvBndtyZ2vfcnSTeU+pV+jPT5zBYXFxk3DvVdSU3kyca6Wq59Uj8cuHUDjBslc+/e57M4rDDukw7Jh534mfraGnwzsRNfWjcMOx5XDk4lzdUCbpqk8dukAvt22jzteW5hQE0I+9t5yDOOG4T3DDsUdgicT5+qIk7u34t/O6cU/v9zI0x+tCjucuKzdto+X5qzlkhM706lFo7DDcYfgycS5OuTq07tzdp+23Pv2N8xZvS3scCr06Ixl1Ksnxg31eyU1nScT5+oQSdx/cT86tWjI9c/PI3d3ftghlWtl7h5em5fD5ScdQ7tmqWGH4yrgycS5OqZpan0mXD6IXXmF3PDCPIqKS8IOqUyPzFhGg+QkrjuzR9ihuDh4MnGuDjq2fVN+d8EJzF65jfvfWVrxDtVs6abdTP5iPVee0pX0tAZhh+Pi4MnEuTrqokGduPSkLjzxwQreWbQx7HAO8ND0pTROSeaa07uHHYqLkycT5+qw35zXh76dmnHbK1+wesvesMMB4Kt1O3n7q4384rRutGicEnY4Lk6eTJyrw1LrJzH+0oHUk7j2ubnsLwh/QsiHpi+laWoyV53WLexQ3GHwZOJcHde5ZSMeHtOfJZt2819vfhXqC43z12xnxjebufr07jRrWD+0ONzh82TinGNorzbcMCyDV+fm8NKctRXvUEUenL6Ulo1T+Nmp3itJNJ5MnHMA3DQ8gyEZrfnN5EV8mbOz2o//+aptzFq2hWvP6E6TBsnVfnxXOZ5MnHMAJNUTj4wZQOvGKVz3/Fx27CuotmObGfe/s4T0tAZccXLXajuuO3o8mTjnvtOycQrjLxvIpl153PryF5SUVM/9k4+Xb+XzVdu4/sweNExJqpZjuqMrrmQiaaSkJZKWS7qznDoXS1osaZGkiVHlUyXtkDQlpv6sqBUT10uaFJRfJmlh8PWJpH4VxREsKfyZpGWSXgqWF3bOHYEBXVrwm/P68N43m3n8/eVVfjwz44HpS2jfLJUxg7tU+fFc1agwmUhKAsYD5wJ9gLGS+sTUyQDuAk41s+OAm6M230dkud0DmNkQM+tvZv2BT4ms+w6wCjjDzPoC9wBPxhHH74GHzCwD2A5cFce5O+fKcfnJx3BB/w48MH0ps5blVumx3l+Sy/w1O7hhWAap9b1Xkqji6ZkMBpab2UozKwBeBEbF1PkVMN7MtgOY2ebSDWY2Ayh3eTdJacAwYFJQ/5PSdoDZQKdDxSFJwf6vBvX+ClwQx3k558ohif/98QlktGnCTS8uYP2O/VVynNJeSeeWDRmd1aniHVyNFU8y6QhEPyuYE5RFywQyJX0sabakkYcRw4XADDPbVca2q4C3K4ijFbDDzIoOER8Akq6WlC0pOze3an/bci7RNUpJZsLlg8gvLOb6ifMoKDr6E0JOW7SJr9bt4qbhmdRP8lu4iSyeq6cyymLvyiUDGcCZwFjgKUnN44xhLPDCQQeVhhJJJndUEEc88UUKzZ40sywzy0pPT48zPOfqrh7pTbhvdD/mr9nB//7z66PadkmJ8dD0pXRv3ZgL+nc4qm276hdPMskBOkd97gSsL6POm2ZWaGargCVEksshSWpFZPjqrZjyvsBTwCgz21pBHFuA5pKSY8qdc0fBD09ozy9P68azn6zmzQXrjlq7U77cwJJNu7nprAySvVeS8OK5gnOAjOCJqRRgDDA5ps4kYCiApNZEhr1WxtH2aGCKmeWVFkjqQuRm/BVmFj03dplxWGTuh5nAT4J6VwJvxnFs51yc7ji3Nyd2bcGdr33Jsk3l3gKNW1FxCQ+/u5RebdP4l77eK6kNKkwmwb2IccA04GvgZTNbJOluSecH1aYBWyUtJvKD/fbSHoWkWcArwHBJOZLOiWp+DAcPcf2GyH2Qx4PHhrMPFUewzx3ArZKWB/s+fVjfBefcIdVPqsdjlw6kcYMkrn1uLnvyiyre6RAmLVjPyty93DIig3r1yhqpdolGYU7qFqasrCzLzs4OOwznEsqnK7Zy2VOzOfeE9jw2dgCRhykPT2FxCcMf+IC01GSm3HDaEbXhwiNprpllxZb7QKVzLm4/6NGK28/pzVsLN/CXj1cfURuvzs1hzbZ93HZ2pieSWsSTiXPusFx7RndG9GnL//7za7JXbzusffOLivnjjGX079ycob3aVFGELgyeTJxzh0US94/uR8cWDbl+4jy27MmPe98XP1/L+p15/OvZvbxXUst4MnHOHbZmDesz4bJB7NhXyA0T51NUXPELjfsLinls5nIGd2vJqT1bVUOUrjp5MnHOHZE+HZry/y44nk9XbuXB6UsrrP/c7G/J3Z3PbSP8Xklt5MnEOXfERmd1Zuzgzjz+/gqmL95Ubr29+UVM+GAFQzJac1J375XURp5MnHOV8t//chzHd2zKrS8vYM3WfWXWefaT1WzbW8CtIzKrOTpXXTyZOOcqJbV+EhMuG0Q9iWufm0teYfEB23fuL+RPH6xgeO82DOjSIqQoXVXzZOKcq7TOLRvx0CX9WLxhF79586sDtj390Sp25RVxi/dKajVPJs65o2JY77bcMKwnL2fn8NKcNQBs31vAMx+tYuRx7Ti+Y7OQI3RVKbniKs45F5+bz8pk/pod/NebiziuQzPe+nIDewu8V1IXeM/EOXfUJNUTj4zpT6vGKVz73Fye/Xg1/9K3A73apYUdmqtinkycc0dVqyYNGH/ZQDbtyiO/qJibz6pwaSNXC/gwl3PuqBvYpQXjLx3I9n0FdE9vEnY4rhp4MnHOVYmzj2sXdgiuGvkwl3POuUrzZOKcc67S4komkkZKWiJpuaQ7y6lzsaTFkhZJmhhVPlXSDklTYurPCpblXSBpvaRJQXlvSZ9Kypf0r1H1e0XVXyBpl6Sbg22/lbQuatsPj+Sb4Zxz7shUeM9EUhIwHhgB5ABzJE02s8VRdTKAu4BTzWy7pOhVb+4DGgHXRLdrZkOi9n8NeDP4uA24Ebggpv4SoH9UTOuAN6KqPGRm91d0Ps45546+eHomg4HlZrbSzAqAF4FRMXV+BYw3s+0AZra5dIOZzQB2l9e4pDRgGDCpdF8zmwMUHiKm4cAKM/s2jvidc85VsXiSSUdgbdTnnKAsWiaQKeljSbMljTyMGC4EZpjZrsPYZwzwQkzZOEkLJT0jqczZ5CRdLSlbUnZubu5hHM4559yhxJNMylrFxmI+JwMZwJnAWOApSc3jjGEsByeG8oORUoDzgVeiiicAPYgMg20AHihrXzN70syyzCwrPT093kM655yrQDzJJAfoHPW5E7C+jDpvmlmhma0ClhBJLockqRWRYbS34gsXgHOBeWb23Uo8ZrbJzIrNrAT4c9Cmc865ahLPS4tzgAxJ3Yjc9B4DXBpTZxKRHsazkloTGfZaGUfbo4EpZpYXf8gH92QktTezDcHHC4GvDtorxty5c7dIOtJ7Lq2BLUe4b03j51Lz1JbzAD+Xmqoy53JMWYUyix2xKqNS5FHbh4Ek4Bkz+52ku4FsM5usyILODwAjgWLgd2b2YrDvLKA30ATYClxlZtOCbe8D95rZ1KhjtQOygaZACbAH6GNmuyQ1InL/pruZ7Yza5+9EhrgMWA1cE5VcjjpJ2WaWVVXtVyc/l5qntpwH+LnUVFVxLnElE3cg/0dVM9WWc6kt5wF+LjVVVZyLvwHvnHOu0jyZHJknww7gKPJzqXlqy3mAn0tNddTPxYe5nHPOVZr3TJxzzlWaJxPnnHOV5skkDpJWS/oymJE4OyhrKWm6pGXBn2VO4VKTlHMeCTnjsqTmkl6V9I2kryX9IBGvCZR7Lgl3Xcqb2TvRrsshziPhrgmApFsUmc39K0kvSEqV1E3SZ8E1eSmYWaRyx/F7JhWTtBrIMrMtUWV/ALaZ2b2KTMvfwszuCCvGeJRzHr8F9iTajMuS/grMMrOngv8IjYB/J8GuCZR7LjeTgNelVNTM3icB15OA1wUOOo+fk2DXRFJH4CMi7+rtl/Qy8E/gh8DrZvaipCeAL8xsQmWO5T2TIzcK+Gvw978SM2W+qzqSmgKnA08DmFmBme0gAa/JIc4l0UXP7J1w1yVKbZihPBloKCmZyC8qG4jM1P5qsP2oXBNPJvEx4B1JcyVdHZS1LX3LPvizTbl71xxlnQfEMeNyDdMdyAX+Imm+pKckNSYxr0l55wKJd12iRc/snYjXpVTsDOUJdU3MbB1wP7CGSBLZCcwFdphZUVCtrJngD5snk/icamYDiUwyeb2k08MO6AiVdR5xzbhcwyQDA4EJZjYA2AuUuQJoAijvXBLxugDlzuydcMo4j4S7JkHCGwV0AzoAjYn8/49V6fsdnkziYGbrgz83E1ndcTCwSVJ7iEw0CWwuv4WaoazzSNAZl3OAHDP7LPj8KpEfyAl3TSjnXBL0upSKndk7Ea8LxJxHgl6Ts4BVZpZrZoXA68ApQPNg2AvKngn+sHkyqYCkxoqsBkkw/HA2kVmJJwNXBtWu5Ptlh2uk8s6j9D95IK4Zl8NmZhuBtZJ6BUXDgcUk2DWB8s8lEa9LlNiZvRPuugQOOI8EvSZrgJMlNZIkvv+/MhP4SVDnqFwTf5qrApK68/1a88nAxGDW5FbAy0AXIhdstJltCynMCh3iPKp1xuWjRVJ/4CkghchyBz8n8stRwlyTUuWcy6Mk5nU5aGbvRPu/AuWeR6L+X/kf4BKgCJgP/JLIPZIXgZZB2eVmll+p43gycc45V1k+zOWcc67SPJk455yrNE8mzjnnKs2TiXPOuUrzZOKcc67SPJk455yrNE8mzjnnKu3/Aya9vMAwLNaPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# # plot CV误差曲线\n",
    "test_means = grid_search_min_child_samples_v3.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search_min_child_samples_v3.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid_search_v1.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid_search_v1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "x_axis = min_child_samples_s\n",
    "\n",
    "plt.plot(x_axis, -test_means)\n",
    "# #plt.errorbar(x_axis, -test_scores, yerr=test_stds ,label = ' Test')\n",
    "# #plt.errorbar(x_axis, -train_scores, yerr=train_stds,label =  +' Train')\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### min_child_samples=70"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### end 有时间需要对下面的行采样参数、列采样参数 进行同样的网格搜索计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.01,\n",
    "          'min_child_samples':70, # 每个叶子节点的最小样本数目\n",
    "          'n_estimators':10000,\n",
    "          'max_depth': 7,\n",
    "          'max_bin': 64, # 稀疏离散特征一共有167维，先不设置很大，提高训练速度\n",
    "          'subsample': 0.7,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.7,\n",
    "          'num_leaves': 30,\n",
    "         }\n",
    "lg_v4 = LGBMRegressor(silent=False,  **params)\n",
    "lg_v4.fit(train_X, train_Y)\n",
    "try:\n",
    "    # 序列化到磁盘中\n",
    "    with open(model_path + 'lg_v4.pkl','wb') as fw:\n",
    "        pk.dump(lg_v4,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg_v4 error,',e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 行采样参数 sub_samples/bagging_fraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7, # 树最大深度\n",
    "          'num_leaves':30, # 叶子节点数目\n",
    "          'min_child_samples':70, # 每个叶子节点的最小样本数目\n",
    "          'max_bin': 64, # 计算特征直方图时的bin 的最大数量\n",
    "          #'subsample': 0.7,\n",
    "          'bagging_freq': 1, # bagging 的频率，表示每 K 次执行一次bagging\n",
    "          'colsample_bytree': 0.7, # 列采样比例，建每棵树使用的样本字段比例\n",
    "          'sparse_threshold': 0.8  # 稀疏比例阈值(0值的比例超过该值为稀疏特征)\n",
    "          'early_stopping_round':10 # 添加 early_stopping_round，减少迭代次数\n",
    "         }\n",
    "lg_v2 = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "subsample_s = [i/10.0 for i in range(6,10)]\n",
    "tuned_parameters = dict( subsample = subsample_s)\n",
    "\n",
    "grid_search_v2 = GridSearchCV(lg_v2, n_jobs=6,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search_v2.fit(train_X , train_Y)\n",
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(model_path + 'grid_search_v2.pkl','wb') as fw:\n",
    "        pk.dump(grid_search_v2,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump grid_search_v2 error,',e)\n",
    "try:\n",
    "    # 序列化到磁盘中\n",
    "    with open(model_path + 'lg_v2.pkl','wb') as fw:\n",
    "        pk.dump(lg_v2,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg_v2 error,',e)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # examine the best model\n",
    "# print(-grid_search.best_score_)\n",
    "# print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # plot CV误差曲线\n",
    "# test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "# test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# x_axis = subsample_s\n",
    "\n",
    "# plt.plot(x_axis, -test_means)\n",
    "# #plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = str(max_depths[i]) +' Test')\n",
    "# #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = str(max_depths[i]) +' Train')\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### subsample=0.8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 列采样参数 sub_feature/feature_fraction/colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'regression',\n",
    "          'n_jobs': 6,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':best_n_estimators,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':60,\n",
    "          'min_child_samples':30,\n",
    "          'max_bin': 64, #2^6,原始特征为整数，很少超过100\n",
    "          'subsample': 0.8,  \n",
    "          'bagging_freq': 1,\n",
    "          #'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg_v3 = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "colsample_bytree_s = [i/10.0 for i in range(6,10)]\n",
    "tuned_parameters = dict( colsample_bytree = colsample_bytree_s)\n",
    "\n",
    "grid_search_3 = GridSearchCV(lg_v3, n_jobs=6,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search_3.fit(train_X , train_Y)\n",
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(model_path + 'grid_search_v3.pkl','wb') as fw:\n",
    "        pk.dump(grid_search_v3,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump grid_search_v3 error,',e)\n",
    "try:\n",
    "    # 序列化到磁盘中\n",
    "    with open(model_path + 'lg_v3.pkl','wb') as fw:\n",
    "        pk.dump(lg_v3,fw)\n",
    "    fw.close()\n",
    "except Exception as e:\n",
    "    print('dump lg_v3 error,',e)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # examine the best model\n",
    "# print(-grid_search.best_score_)\n",
    "# print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # plot CV误差曲线\n",
    "# test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "# test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "# train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# x_axis = colsample_bytree_s\n",
    "\n",
    "# plt.plot(x_axis, -test_means)\n",
    "# #plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = str(max_depths[i]) +' Test')\n",
    "# #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = str(max_depths[i]) +' Train')\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('work done!!!!!!hubo7')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "nowTime=datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')#现在\n",
    "print(nowTime)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "再调小一点，由于特征包括原始特征+tfidf特征，是多了些"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# params = {'boosting_type': 'gbdt',\n",
    "#           'objective': 'multiclass',\n",
    "#           'num_class':9, \n",
    "#           'n_jobs': 2,\n",
    "#           'learning_rate': 0.1,\n",
    "#           'n_estimators':n_estimators_1,\n",
    "#           'max_depth': 7,\n",
    "#           'num_leaves':70,\n",
    "#           'min_child_samples':30,\n",
    "#           'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "#           'subsample': 0.8,\n",
    "#           'bagging_freq': 1,\n",
    "#           #'colsample_bytree': 0.7,\n",
    "#          }\n",
    "# lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "# colsample_bytree_s = [i/10.0 for i in range(3,5)]\n",
    "# tuned_parameters = dict( colsample_bytree = colsample_bytree_s)\n",
    "\n",
    "# grid_search = GridSearchCV(lg, n_jobs=4,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "# grid_search.fit(X_train , y_train)\n",
    "# #grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # examine the best model\n",
    "# print(-grid_search.best_score_)\n",
    "# print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### colsample_bytree=0.4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化参数lambda_l1(reg_alpha), lambda_l2(reg_lambda)感觉不用调了"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 减小学习率，调整n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'multiclass',\n",
    "          'num_class':9, \n",
    "          'n_jobs': 2,\n",
    "          'learning_rate': 0.01,\n",
    "          #'n_estimators':n_estimators_1,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':70,\n",
    "          'min_child_samples':30,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'subsample': 0.8,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.4,\n",
    "         }\n",
    "n_estimators_2 = get_n_estimators(params , X_train , y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用所有训练数据，采用最佳参数重新训练模型\n",
    "由于样本数目增多，模型复杂度稍微扩大一点？\n",
    "num_leaves增多5\n",
    "min_child_samples按样本比例增加到40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'multiclass',\n",
    "          'n_jobs': 2,\n",
    "          'learning_rate': 0.01,\n",
    "          'n_estimators':n_estimators_2,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':75,\n",
    "          'min_child_samples':40,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'subsample': 0.8,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.4,\n",
    "         }\n",
    "\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "lg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存模型，用于后续测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cPickle\n",
    "\n",
    "cPickle.dump(lg, open(\"Otto_LightGBM_org_tfidf.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 特征重要性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\"columns\":list(feat_names), \"importance\":list(lg.feature_importances_.T)})\n",
    "df = df.sort_values(by=['importance'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.bar(range(len(lg.feature_importances_)), lg.feature_importances_)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tfidf的特征重要性更高一些。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
